From 4754954bcda6e18709fe00cf8bbde6318abd80ef Mon Sep 17 00:00:00 2001 From: Heiko Rothkranz Date: Fri, 5 Nov 2021 22:10:20 +0800 Subject: [PATCH] feat(core): Support CSV import in multiple languages (#1199) --- .../developer-guide/importing-product-data.md | 6 + .../e2e/__snapshots__/import.e2e-spec.ts.snap | 106 +++ .../e2e-product-import-multi-languages.csv | 4 + packages/core/e2e/import.e2e-spec.ts | 153 ++++ .../__snapshots__/import-parser.spec.ts.snap | 856 +++++++++++++----- .../import-parser/import-parser.spec.ts | 60 +- .../providers/import-parser/import-parser.ts | 548 ++++++++--- .../test-fixtures/multiple-languages.csv | 4 + .../providers/importer/importer.ts | 155 ++-- 9 files changed, 1504 insertions(+), 388 deletions(-) create mode 100644 packages/core/e2e/fixtures/e2e-product-import-multi-languages.csv create mode 100644 packages/core/src/data-import/providers/import-parser/test-fixtures/multiple-languages.csv diff --git a/docs/content/developer-guide/importing-product-data.md b/docs/content/developer-guide/importing-product-data.md index 2bdec6336f..7b026b1c15 100644 --- a/docs/content/developer-guide/importing-product-data.md +++ b/docs/content/developer-guide/importing-product-data.md @@ -76,6 +76,12 @@ To import custom fields with `list` set to `true`, the data should be separated ... ,tablet|pad|android ``` +#### Importing data in multiple languages + +If a field is translatable (i.e. of `localeString` type), you can use column names with an appended language code (e.g. `name:en`, `name:de`, `product:keywords:en`, `product:keywords:de`) to specify its value in multiple languages. + +Use of language codes has to be consistent throughout the file. You don't have to translate every translatable field. If there are no translated columns for a field, the generic column's value will be used for all languages. But when you do translate columns, the set of languages for each of them needs to be the same. As an example, you cannot use `name:en` and `name:de`, but only provide `slug:en` (it's okay to use only a `slug` column though, in which case this slug will be used for both the English and the German version). + ## Initial Data As well as product data, other initialization data can be populated using the [`InitialData` object]({{< relref "initial-data" >}}). **This format is intentionally limited**; more advanced requirements (e.g. setting up ShippingMethods that use custom checkers & calculators) should be carried out via scripts which interact with the [Admin GraphQL API]({{< relref "/docs/graphql-api/admin" >}}). diff --git a/packages/core/e2e/__snapshots__/import.e2e-spec.ts.snap b/packages/core/e2e/__snapshots__/import.e2e-spec.ts.snap index a894bdb17c..3e2c554ec2 100644 --- a/packages/core/e2e/__snapshots__/import.e2e-spec.ts.snap +++ b/packages/core/e2e/__snapshots__/import.e2e-spec.ts.snap @@ -371,3 +371,109 @@ Object { ], } `; + +exports[`Import resolver imports products with multiple languages 1`] = ` +Object { + "assets": Array [], + "customFields": Object { + "keywords": Array [ + "paper, stretch", + ], + "localName": "纸张拉伸器", + "owner": null, + "pageType": null, + }, + "description": "一个用于拉伸纸张的伟大装置", + "featuredAsset": null, + "id": "T_5", + "name": "奇妙的纸张拉伸器", + "optionGroups": Array [ + Object { + "code": "fantastic-paper-stretcher-size", + "id": "T_5", + "name": "size", + }, + ], + "slug": "奇妙的纸张拉伸器", + "variants": Array [ + Object { + "assets": Array [], + "customFields": Object { + "weight": 243, + }, + "featuredAsset": null, + "id": "T_11", + "name": "奇妙的纸张拉伸器 半英制", + "price": 4530, + "sku": "PPS12", + "stockMovements": Object { + "items": Array [ + Object { + "id": "T_3", + "quantity": 10, + "type": "ADJUSTMENT", + }, + ], + }, + "stockOnHand": 10, + "taxCategory": Object { + "id": "T_1", + "name": "Standard Tax", + }, + "trackInventory": "FALSE", + }, + Object { + "assets": Array [], + "customFields": Object { + "weight": 344, + }, + "featuredAsset": null, + "id": "T_12", + "name": "奇妙的纸张拉伸器 四分之一英制", + "price": 3250, + "sku": "PPS14", + "stockMovements": Object { + "items": Array [ + Object { + "id": "T_4", + "quantity": 10, + "type": "ADJUSTMENT", + }, + ], + }, + "stockOnHand": 10, + "taxCategory": Object { + "id": "T_1", + "name": "Standard Tax", + }, + "trackInventory": "FALSE", + }, + Object { + "assets": Array [], + "customFields": Object { + "weight": 656, + }, + "featuredAsset": null, + "id": "T_13", + "name": "奇妙的纸张拉伸器 全英制", + "price": 5950, + "sku": "PPSF", + "stockMovements": Object { + "items": Array [ + Object { + "id": "T_5", + "quantity": 10, + "type": "ADJUSTMENT", + }, + ], + }, + "stockOnHand": 10, + "taxCategory": Object { + "id": "T_1", + "name": "Standard Tax", + }, + "trackInventory": "FALSE", + }, + ], +} +`; diff --git a/packages/core/e2e/fixtures/e2e-product-import-multi-languages.csv b/packages/core/e2e/fixtures/e2e-product-import-multi-languages.csv new file mode 100644 index 0000000000..5c456365af --- /dev/null +++ b/packages/core/e2e/fixtures/e2e-product-import-multi-languages.csv @@ -0,0 +1,4 @@ +name:en , name:zh_Hans , slug , description:en , description:zh_Hans , assets , facets:en , facets:zh_Hans , optionGroups , optionValues:en , optionValues:zh_Hans , sku , price , taxCategory , stockOnHand , trackInventory , variantAssets , variantFacets , product:keywords , product:localName:en , product:localName:zh_Hans , variant:weight +Fantastic Paper Stretcher , 奇妙的纸张拉伸器 , , A great device for stretching paper. , 一个用于拉伸纸张的伟大装置 , , make:KB|group:Accessory , 品牌:KB|类型:饰品 , size , Half Imperial , 半英制 , PPS12 , 45.3 , standard , 10 , false , , , "paper, stretch" , Paper Stretcher , 纸张拉伸器 , 243 + , , , , , , , , , Quarter Imperial , 四分之一英制 , PPS14 , 32.5 , standard , 10 , false , , , , , , 344 + , , , , , , , , , Full Imperial , 全英制 , PPSF , 59.5 , standard , 10 , false , , , , , , 656 diff --git a/packages/core/e2e/import.e2e-spec.ts b/packages/core/e2e/import.e2e-spec.ts index 91f2a1fc9c..93de6ab866 100644 --- a/packages/core/e2e/import.e2e-spec.ts +++ b/packages/core/e2e/import.e2e-spec.ts @@ -247,4 +247,157 @@ describe('Import resolver', () => { expect(pencils.customFields.localName).toEqual('localGiotto'); expect(smock.customFields.localName).toEqual('localSmock'); }, 20000); + + it('imports products with multiple languages', async () => { + // TODO: see test above + const timeout = process.env.CI ? 2000 : 1000; + await new Promise(resolve => { + setTimeout(resolve, timeout); + }); + + const csvFile = path.join(__dirname, 'fixtures', 'e2e-product-import-multi-languages.csv'); + const result = await adminClient.fileUploadMutation({ + mutation: gql` + mutation ImportProducts($csvFile: Upload!) { + importProducts(csvFile: $csvFile) { + imported + processed + errors + } + } + `, + filePaths: [csvFile], + mapVariables: () => ({ csvFile: null }), + }); + + expect(result.importProducts.errors).toEqual([]); + expect(result.importProducts.imported).toBe(1); + expect(result.importProducts.processed).toBe(1); + + const productResult = await adminClient.query( + gql` + query GetProducts($options: ProductListOptions) { + products(options: $options) { + totalItems + items { + id + name + slug + description + featuredAsset { + id + name + preview + source + } + assets { + id + name + preview + source + } + optionGroups { + id + code + name + } + facetValues { + id + name + facet { + id + name + } + } + customFields { + pageType + owner { + id + } + keywords + localName + } + variants { + id + name + sku + price + taxCategory { + id + name + } + options { + id + code + name + } + assets { + id + name + preview + source + } + featuredAsset { + id + name + preview + source + } + facetValues { + id + code + name + facet { + id + name + } + } + stockOnHand + trackInventory + stockMovements { + items { + ... on StockMovement { + id + type + quantity + } + } + } + customFields { + weight + } + } + } + } + } + `, + { + options: {}, + }, + { + languageCode: 'zh_Hans', + }, + ); + + expect(productResult.products.totalItems).toBe(5); + + const paperStretcher = productResult.products.items.find((p: any) => p.name === '奇妙的纸张拉伸器'); + + // Omit FacetValues & options due to variations in the ordering between different DB engines + expect(omit(paperStretcher, ['facetValues', 'options'], true)).toMatchSnapshot(); + + const byName = (e: { name: string }) => e.name; + + expect(paperStretcher.facetValues.map(byName).sort()).toEqual(['KB', '饰品']); + + expect(paperStretcher.variants[0].options.map(byName).sort()).toEqual(['半英制']); + expect(paperStretcher.variants[1].options.map(byName).sort()).toEqual(['四分之一英制']); + expect(paperStretcher.variants[2].options.map(byName).sort()).toEqual(['全英制']); + + // Import list custom fields + expect(paperStretcher.customFields.keywords).toEqual(['paper, stretch']); + + // Import localeString custom fields + expect(paperStretcher.customFields.localName).toEqual('纸张拉伸器'); + }, 20000); }); diff --git a/packages/core/src/data-import/providers/import-parser/__snapshots__/import-parser.spec.ts.snap b/packages/core/src/data-import/providers/import-parser/__snapshots__/import-parser.spec.ts.snap index fbe24b5fab..9ec1c434fd 100644 --- a/packages/core/src/data-import/providers/import-parser/__snapshots__/import-parser.spec.ts.snap +++ b/packages/core/src/data-import/providers/import-parser/__snapshots__/import-parser.spec.ts.snap @@ -5,70 +5,95 @@ Array [ Object { "product": Object { "assetPaths": Array [], - "customFields": Object { - "customPage": "grid-view", - "keywords": "paper, stretch", - }, - "description": "A great device for stretching paper.", "facets": Array [], - "name": "Perfect Paper Stretcher", "optionGroups": Array [ Object { - "name": "size", - "values": Array [ - "Half Imperial", - "Quarter Imperial", - "Full Imperial", + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "Half Imperial", + "Quarter Imperial", + "Full Imperial", + ], + }, ], }, ], - "slug": "perfect-paper-stretcher", + "translations": Array [ + Object { + "customFields": Object { + "customPage": "grid-view", + "keywords": "paper, stretch", + }, + "description": "A great device for stretching paper.", + "languageCode": "en", + "name": "Perfect Paper Stretcher", + "slug": "perfect-paper-stretcher", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object { - "volumetric": "243", - }, "facets": Array [], - "optionValues": Array [ - "Half Imperial", - ], "price": 45.3, "sku": "PPS12", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object { + "volumetric": "243", + }, + "languageCode": "en", + "optionValues": Array [ + "Half Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object { - "volumetric": "344", - }, "facets": Array [], - "optionValues": Array [ - "Quarter Imperial", - ], "price": 32.5, "sku": "PPS14", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object { + "volumetric": "344", + }, + "languageCode": "en", + "optionValues": Array [ + "Quarter Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object { - "volumetric": "656", - }, "facets": Array [], - "optionValues": Array [ - "Full Imperial", - ], "price": 59.5, "sku": "PPSF", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object { + "volumetric": "656", + }, + "languageCode": "en", + "optionValues": Array [ + "Full Imperial", + ], + }, + ], }, ], }, @@ -80,216 +105,306 @@ Array [ Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "A great device for stretching paper.", "facets": Array [], - "name": "Perfect Paper Stretcher", "optionGroups": Array [ Object { - "name": "size", - "values": Array [ - "Half Imperial", - "Quarter Imperial", - "Full Imperial", + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "Half Imperial", + "Quarter Imperial", + "Full Imperial", + ], + }, ], }, ], - "slug": "Perfect-paper-stretcher", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "A great device for stretching paper.", + "languageCode": "en", + "name": "Perfect Paper Stretcher", + "slug": "Perfect-paper-stretcher", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Half Imperial", - ], "price": 45.3, "sku": "PPS12", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Half Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Quarter Imperial", - ], "price": 32.5, "sku": "PPS14", "stockOnHand": 11, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Quarter Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Full Imperial", - ], "price": 59.5, "sku": "PPSF", "stockOnHand": 12, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Full Imperial", + ], + }, + ], }, ], }, Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "Mabef description", "facets": Array [], - "name": "Mabef M/02 Studio Easel", "optionGroups": Array [], - "slug": "mabef-m02-studio-easel", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "Mabef description", + "languageCode": "en", + "name": "Mabef M/02 Studio Easel", + "slug": "mabef-m02-studio-easel", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [], "price": 910.7, "sku": "M02", "stockOnHand": 13, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [], + }, + ], }, ], }, Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "Really mega pencils", "facets": Array [], - "name": "Giotto Mega Pencils", "optionGroups": Array [ Object { - "name": "box size", - "values": Array [ - "Box of 8", - "Box of 12", + "translations": Array [ + Object { + "languageCode": "en", + "name": "box size", + "values": Array [ + "Box of 8", + "Box of 12", + ], + }, ], }, ], - "slug": "giotto-mega-pencils", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "Really mega pencils", + "languageCode": "en", + "name": "Giotto Mega Pencils", + "slug": "giotto-mega-pencils", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Box of 8", - ], "price": 4.16, "sku": "225400", "stockOnHand": 14, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Box of 8", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Box of 12", - ], "price": 6.24, "sku": "225600", "stockOnHand": 15, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Box of 12", + ], + }, + ], }, ], }, Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "Keeps the paint off the clothes", "facets": Array [], - "name": "Artists Smock", "optionGroups": Array [ Object { - "name": "size", - "values": Array [ - "small", - "large", + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "small", + "large", + ], + }, ], }, Object { - "name": "colour", - "values": Array [ - "beige", - "navy", + "translations": Array [ + Object { + "languageCode": "en", + "name": "colour", + "values": Array [ + "beige", + "navy", + ], + }, ], }, ], - "slug": "artists-smock", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "Keeps the paint off the clothes", + "languageCode": "en", + "name": "Artists Smock", + "slug": "artists-smock", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "small", - "beige", - ], "price": 11.99, "sku": "10112", "stockOnHand": 16, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "small", + "beige", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "large", - "beige", - ], "price": 11.99, "sku": "10113", "stockOnHand": 17, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "large", + "beige", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "small", - "navy", - ], "price": 11.99, "sku": "10114", "stockOnHand": 18, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "small", + "navy", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "large", - "navy", - ], "price": 11.99, "sku": "10115", "stockOnHand": 19, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "large", + "navy", + ], + }, + ], }, ], }, @@ -301,61 +416,86 @@ Array [ Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "A great device for stretching paper.", "facets": Array [], - "name": "Perfect Paper Stretcher", "optionGroups": Array [ Object { - "name": "size", - "values": Array [ - "Half Imperial", - "Quarter Imperial", - "Full Imperial", + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "Half Imperial", + "Quarter Imperial", + "Full Imperial", + ], + }, ], }, ], - "slug": "perfect-paper-stretcher", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "A great device for stretching paper.", + "languageCode": "en", + "name": "Perfect Paper Stretcher", + "slug": "perfect-paper-stretcher", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Half Imperial", - ], "price": 45.3, "sku": "PPS12", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Half Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Quarter Imperial", - ], "price": 32.5, "sku": "PPS14", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Quarter Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Full Imperial", - ], "price": 59.5, "sku": "PPSF", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Full Imperial", + ], + }, + ], }, ], }, @@ -370,38 +510,238 @@ Array [ "pps1.jpg", "pps2.jpg", ], - "customFields": Object {}, - "description": "A great device for stretching paper.", "facets": Array [ Object { - "facet": "brand", - "value": "KB", + "translations": Array [ + Object { + "facet": "brand", + "languageCode": "en", + "value": "KB", + }, + ], }, Object { - "facet": "type", - "value": "Accessory", + "translations": Array [ + Object { + "facet": "type", + "languageCode": "en", + "value": "Accessory", + }, + ], }, ], - "name": "Perfect Paper Stretcher", "optionGroups": Array [], - "slug": "perfect-paper-stretcher", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "A great device for stretching paper.", + "languageCode": "en", + "name": "Perfect Paper Stretcher", + "slug": "perfect-paper-stretcher", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [ Object { - "facet": "material", - "value": "Wood", + "translations": Array [ + Object { + "facet": "material", + "languageCode": "en", + "value": "Wood", + }, + ], }, ], - "optionValues": Array [], "price": 45.3, "sku": "PPS12", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [], + }, + ], + }, + ], + }, +] +`; + +exports[`ImportParser parseProducts works with multilingual input 1`] = ` +Array [ + Object { + "product": Object { + "assetPaths": Array [], + "facets": Array [ + Object { + "translations": Array [ + Object { + "facet": "brand", + "languageCode": "en", + "value": "KB", + }, + Object { + "facet": "品牌", + "languageCode": "zh_Hans", + "value": "KB", + }, + ], + }, + Object { + "translations": Array [ + Object { + "facet": "type", + "languageCode": "en", + "value": "Accessory", + }, + Object { + "facet": "类型", + "languageCode": "zh_Hans", + "value": "饰品", + }, + ], + }, + ], + "optionGroups": Array [ + Object { + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "Half Imperial", + "Quarter Imperial", + "Full Imperial", + ], + }, + Object { + "languageCode": "zh_Hans", + "name": "size", + "values": Array [ + "半英制", + "四分之一英制", + "全英制", + ], + }, + ], + }, + ], + "translations": Array [ + Object { + "customFields": Object { + "customPage": "grid-view", + "keywords": "paper, stretch", + }, + "description": "A great device for stretching paper.", + "languageCode": "en", + "name": "Perfect Paper Stretcher", + "slug": "perfect-paper-stretcher", + }, + Object { + "customFields": Object { + "customPage": "grid-view", + "keywords": "纸张，拉伸", + }, + "description": "一个用于拉伸纸张的伟大装置", + "languageCode": "zh_Hans", + "name": "完美的纸张拉伸器", + "slug": "完美的纸张拉伸器", + }, + ], + }, + "variants": Array [ + Object { + "assetPaths": Array [], + "facets": Array [], + "price": 45.3, + "sku": "PPS12", + "stockOnHand": 10, + "taxCategory": "standard", + "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object { + "volumetric": "243", + }, + "languageCode": "en", + "optionValues": Array [ + "Half Imperial", + ], + }, + Object { + "customFields": Object { + "volumetric": "243", + }, + "languageCode": "zh_Hans", + "optionValues": Array [ + "半英制", + ], + }, + ], + }, + Object { + "assetPaths": Array [], + "facets": Array [], + "price": 32.5, + "sku": "PPS14", + "stockOnHand": 10, + "taxCategory": "standard", + "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object { + "volumetric": "344", + }, + "languageCode": "en", + "optionValues": Array [ + "Quarter Imperial", + ], + }, + Object { + "customFields": Object { + "volumetric": "344", + }, + "languageCode": "zh_Hans", + "optionValues": Array [ + "四分之一英制", + ], + }, + ], + }, + Object { + "assetPaths": Array [], + "facets": Array [], + "price": 59.5, + "sku": "PPSF", + "stockOnHand": 10, + "taxCategory": "standard", + "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object { + "volumetric": "656", + }, + "languageCode": "en", + "optionValues": Array [ + "Full Imperial", + ], + }, + Object { + "customFields": Object { + "volumetric": "656", + }, + "languageCode": "zh_Hans", + "optionValues": Array [ + "全英制", + ], + }, + ], }, ], }, @@ -413,216 +753,306 @@ Array [ Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "A great device for stretching paper.", "facets": Array [], - "name": "Perfect Paper Stretcher", "optionGroups": Array [ Object { - "name": "size", - "values": Array [ - "Half Imperial", - "Quarter Imperial", - "Full Imperial", + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "Half Imperial", + "Quarter Imperial", + "Full Imperial", + ], + }, ], }, ], - "slug": "Perfect-paper-stretcher", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "A great device for stretching paper.", + "languageCode": "en", + "name": "Perfect Paper Stretcher", + "slug": "Perfect-paper-stretcher", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Half Imperial", - ], "price": 45.3, "sku": "PPS12", "stockOnHand": 10, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Half Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Quarter Imperial", - ], "price": 32.5, "sku": "PPS14", "stockOnHand": 11, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Quarter Imperial", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Full Imperial", - ], "price": 59.5, "sku": "PPSF", "stockOnHand": 12, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Full Imperial", + ], + }, + ], }, ], }, Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "Mabef description", "facets": Array [], - "name": "Mabef M/02 Studio Easel", "optionGroups": Array [], - "slug": "mabef-m02-studio-easel", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "Mabef description", + "languageCode": "en", + "name": "Mabef M/02 Studio Easel", + "slug": "mabef-m02-studio-easel", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [], "price": 910.7, "sku": "M02", "stockOnHand": 13, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [], + }, + ], }, ], }, Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "Really mega pencils", "facets": Array [], - "name": "Giotto Mega Pencils", "optionGroups": Array [ Object { - "name": "box size", - "values": Array [ - "Box of 8", - "Box of 12", + "translations": Array [ + Object { + "languageCode": "en", + "name": "box size", + "values": Array [ + "Box of 8", + "Box of 12", + ], + }, ], }, ], - "slug": "giotto-mega-pencils", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "Really mega pencils", + "languageCode": "en", + "name": "Giotto Mega Pencils", + "slug": "giotto-mega-pencils", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Box of 8", - ], "price": 4.16, "sku": "225400", "stockOnHand": 14, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Box of 8", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "Box of 12", - ], "price": 6.24, "sku": "225600", "stockOnHand": 15, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "Box of 12", + ], + }, + ], }, ], }, Object { "product": Object { "assetPaths": Array [], - "customFields": Object {}, - "description": "Keeps the paint off the clothes", "facets": Array [], - "name": "Artists Smock", "optionGroups": Array [ Object { - "name": "size", - "values": Array [ - "small", - "large", + "translations": Array [ + Object { + "languageCode": "en", + "name": "size", + "values": Array [ + "small", + "large", + ], + }, ], }, Object { - "name": "colour", - "values": Array [ - "beige", - "navy", + "translations": Array [ + Object { + "languageCode": "en", + "name": "colour", + "values": Array [ + "beige", + "navy", + ], + }, ], }, ], - "slug": "artists-smock", + "translations": Array [ + Object { + "customFields": Object {}, + "description": "Keeps the paint off the clothes", + "languageCode": "en", + "name": "Artists Smock", + "slug": "artists-smock", + }, + ], }, "variants": Array [ Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "small", - "beige", - ], "price": 11.99, "sku": "10112", "stockOnHand": 16, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "small", + "beige", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "large", - "beige", - ], "price": 11.99, "sku": "10113", "stockOnHand": 17, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "large", + "beige", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "small", - "navy", - ], "price": 11.99, "sku": "10114", "stockOnHand": 18, "taxCategory": "standard", "trackInventory": "FALSE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "small", + "navy", + ], + }, + ], }, Object { "assetPaths": Array [], - "customFields": Object {}, "facets": Array [], - "optionValues": Array [ - "large", - "navy", - ], "price": 11.99, "sku": "10115", "stockOnHand": 19, "taxCategory": "standard", "trackInventory": "TRUE", + "translations": Array [ + Object { + "customFields": Object {}, + "languageCode": "en", + "optionValues": Array [ + "large", + "navy", + ], + }, + ], }, ], }, diff --git a/packages/core/src/data-import/providers/import-parser/import-parser.spec.ts b/packages/core/src/data-import/providers/import-parser/import-parser.spec.ts index 79023999c4..147ae3a92c 100644 --- a/packages/core/src/data-import/providers/import-parser/import-parser.spec.ts +++ b/packages/core/src/data-import/providers/import-parser/import-parser.spec.ts @@ -1,12 +1,38 @@ import fs from 'fs-extra'; import path from 'path'; +import { LanguageCode } from '../../..'; +import { ConfigService } from '../../../config/config.service'; + import { ImportParser } from './import-parser'; +const mockConfigService = { + defaultLanguageCode: LanguageCode.en, + customFields: { + Product: [ + { + name: 'keywords', + type: 'localeString', + list: true, + }, + { + name: 'customPage', + type: 'string', + }, + ], + ProductVariant: [ + { + name: 'volumetric', + type: 'int', + }, + ], + }, +} as ConfigService; + describe('ImportParser', () => { describe('parseProducts', () => { it('single product with a single variant', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('single-product-single-variant.csv'); const result = await importParser.parseProducts(input); @@ -15,7 +41,7 @@ describe('ImportParser', () => { }); it('single product with a multiple variants', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('single-product-multiple-variants.csv'); const result = await importParser.parseProducts(input); @@ -24,7 +50,7 @@ describe('ImportParser', () => { }); it('multiple products with multiple variants', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('multiple-products-multiple-variants.csv'); const result = await importParser.parseProducts(input); @@ -33,7 +59,7 @@ describe('ImportParser', () => { }); it('custom fields', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('custom-fields.csv'); const result = await importParser.parseProducts(input); @@ -42,7 +68,7 @@ describe('ImportParser', () => { }); it('works with streamed input', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const filename = path.join(__dirname, 'test-fixtures', 'multiple-products-multiple-variants.csv'); const input = fs.createReadStream(filename); @@ -51,23 +77,33 @@ describe('ImportParser', () => { expect(result.results).toMatchSnapshot(); }); + it('works with multilingual input', async () => { + const importParser = new ImportParser(mockConfigService); + + const filename = path.join(__dirname, 'test-fixtures', 'multiple-languages.csv'); + const input = fs.createReadStream(filename); + const result = await importParser.parseProducts(input); + + expect(result.results).toMatchSnapshot(); + }); + describe('error conditions', () => { it('reports errors on invalid option values', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('invalid-option-values.csv'); const result = await importParser.parseProducts(input); expect(result.errors).toEqual([ - 'The number of optionValues must match the number of optionGroups on line 2', - 'The number of optionValues must match the number of optionGroups on line 3', - 'The number of optionValues must match the number of optionGroups on line 4', - 'The number of optionValues must match the number of optionGroups on line 5', + "The number of optionValues in column 'optionValues' must match the number of optionGroups on line 2", + "The number of optionValues in column 'optionValues' must match the number of optionGroups on line 3", + "The number of optionValues in column 'optionValues' must match the number of optionGroups on line 4", + "The number of optionValues in column 'optionValues' must match the number of optionGroups on line 5", ]); }); it('reports error on ivalid columns', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('invalid-columns.csv'); const result = await importParser.parseProducts(input); @@ -79,7 +115,7 @@ describe('ImportParser', () => { }); it('reports error on ivalid row length', async () => { - const importParser = new ImportParser(); + const importParser = new ImportParser(mockConfigService); const input = await loadTestFixture('invalid-row-length.csv'); const result = await importParser.parseProducts(input); diff --git a/packages/core/src/data-import/providers/import-parser/import-parser.ts b/packages/core/src/data-import/providers/import-parser/import-parser.ts index 81b2b86b1d..db2901a013 100644 --- a/packages/core/src/data-import/providers/import-parser/import-parser.ts +++ b/packages/core/src/data-import/providers/import-parser/import-parser.ts @@ -1,62 +1,85 @@ import { Injectable } from '@nestjs/common'; -import { GlobalFlag } from '@vendure/common/lib/generated-types'; +import { GlobalFlag, LanguageCode } from '@vendure/common/lib/generated-types'; import { normalizeString } from '@vendure/common/lib/normalize-string'; import { unique } from '@vendure/common/lib/unique'; import parse from 'csv-parse'; import { Stream } from 'stream'; -export type BaseProductRecord = { - name?: string; - slug?: string; - description?: string; - assets?: string; - facets?: string; - optionGroups?: string; - optionValues?: string; - sku?: string; - price?: string; - taxCategory?: string; - stockOnHand?: string; - trackInventory?: string; - variantAssets?: string; - variantFacets?: string; -}; - -export type RawProductRecord = BaseProductRecord & { [customFieldName: string]: string }; +import { InternalServerError } from '../../../common/error/errors'; +import { ConfigService } from '../../../config/config.service'; +import { CustomFieldConfig } from '../../../config/custom-field/custom-field-types'; + +const baseTranslatableColumns = [ + 'name', + 'slug', + 'description', + 'facets', + 'optionGroups', + 'optionValues', + 'variantFacets', +]; + +const requiredColumns: string[] = [ + 'name', + 'slug', + 'description', + 'assets', + 'facets', + 'optionGroups', + 'optionValues', + 'sku', + 'price', + 'taxCategory', + 'variantAssets', + 'variantFacets', +]; + +export interface ParsedOptionGroup { + translations: Array<{ + languageCode: LanguageCode; + name: string; + values: string[]; + }>; +} + +export interface ParsedFacet { + translations: Array<{ + languageCode: LanguageCode; + facet: string; + value: string; + }>; +} export interface ParsedProductVariant { - optionValues: string[]; sku: string; price: number; taxCategory: string; stockOnHand: number; trackInventory: GlobalFlag; assetPaths: string[]; - facets: Array<{ - facet: string; - value: string; + facets: ParsedFacet[]; + translations: Array<{ + languageCode: LanguageCode; + optionValues: string[]; + customFields: { + [name: string]: string; + }; }>; - customFields: { - [name: string]: string; - }; } export interface ParsedProduct { - name: string; - slug: string; - description: string; assetPaths: string[]; - optionGroups: Array<{ + optionGroups: ParsedOptionGroup[]; + facets: ParsedFacet[]; + translations: Array<{ + languageCode: LanguageCode; name: string; - values: string[]; - }>; - facets: Array<{ - facet: string; - value: string; + slug: string; + description: string; + customFields: { + [name: string]: string; + }; }>; - customFields: { - [name: string]: string; - }; } export interface ParsedProductWithVariants { @@ -70,27 +93,17 @@ export interface ParseResult { processed: number; } -const requiredColumns: Array = [ - 'name', - 'slug', - 'description', - 'assets', - 'facets', - 'optionGroups', - 'optionValues', - 'sku', - 'price', - 'taxCategory', - 'variantAssets', - 'variantFacets', -]; - /** * Validates and parses CSV files into a data structure which can then be used to created new entities. */ @Injectable() export class ImportParser { - async parseProducts(input: string | Stream): Promise> { + constructor(private configService: ConfigService) {} + + async parseProducts( + input: string | Stream, + mainLanguage: LanguageCode = this.configService.defaultLanguageCode, + ): Promise> { const options: parse.Options = { trim: true, relax_column_count: true, @@ -104,7 +117,7 @@ export class ImportParser { errors = errors.concat(err); } if (records) { - const parseResult = this.processRawRecords(records); + const parseResult = this.processRawRecords(records, mainLanguage); errors = errors.concat(parseResult.errors); resolve({ results: parseResult.results, errors, processed: parseResult.processed }); } else { @@ -125,7 +138,7 @@ export class ImportParser { }); parser.on('error', reject); parser.on('end', () => { - const parseResult = this.processRawRecords(records); + const parseResult = this.processRawRecords(records, mainLanguage); errors = errors.concat(parseResult.errors); resolve({ results: parseResult.results, errors, processed: parseResult.processed }); }); @@ -133,17 +146,29 @@ export class ImportParser { }); } - private processRawRecords(records: string[][]): ParseResult { + private processRawRecords( + records: string[][], + mainLanguage: LanguageCode, + ): ParseResult { const results: ParsedProductWithVariants[] = []; const errors: string[] = []; let currentRow: ParsedProductWithVariants | undefined; const headerRow = records[0]; const rest = records.slice(1); const totalProducts = rest.map(row => row[0]).filter(name => name.trim() !== '').length; + const customFieldErrors = this.validateCustomFields(headerRow); + if (customFieldErrors.length > 0) { + return { results: [], errors: customFieldErrors, processed: 0 }; + } + const translationError = this.validateHeaderTranslations(headerRow); + if (translationError) { + return { results: [], errors: [translationError], processed: 0 }; + } const columnError = validateRequiredColumns(headerRow); if (columnError) { return { results: [], errors: [columnError], processed: 0 }; } + const usedLanguages = usedLanguageCodes(headerRow); let line = 1; for (const record of rest) { line++; @@ -153,18 +178,18 @@ export class ImportParser { continue; } const r = mapRowToObject(headerRow, record); - if (r.name) { + if (getRawMainTranslation(r, 'name', mainLanguage)) { if (currentRow) { populateOptionGroupValues(currentRow); results.push(currentRow); } currentRow = { - product: parseProductFromRecord(r), - variants: [parseVariantFromRecord(r)], + product: this.parseProductFromRecord(r, usedLanguages, mainLanguage), + variants: [this.parseVariantFromRecord(r, usedLanguages, mainLanguage)], }; } else { if (currentRow) { - currentRow.variants.push(parseVariantFromRecord(r)); + currentRow.variants.push(this.parseVariantFromRecord(r, usedLanguages, mainLanguage)); } } const optionError = validateOptionValueCount(r, currentRow); @@ -178,20 +203,320 @@ export class ImportParser { } return { results, errors, processed: totalProducts }; } + + private validateCustomFields(rowKeys: string[]): string[] { + const errors: string[] = []; + for (const rowKey of rowKeys) { + const baseKey = getBaseKey(rowKey); + const parts = baseKey.split(':'); + if (parts.length === 1) { + continue; + } + if (parts.length === 2) { + let customFieldConfigs: CustomFieldConfig[] = []; + if (parts[0] === 'product') { + customFieldConfigs = this.configService.customFields.Product; + } else if (parts[0] === 'variant') { + customFieldConfigs = this.configService.customFields.ProductVariant; + } else { + continue; + } + const customFieldConfig = customFieldConfigs.find(config => config.name === parts[1]); + if (customFieldConfig) { + continue; + } + } + errors.push(`Invalid custom field: ${rowKey}`); + } + return errors; + } + + private isTranslatable(baseKey: string): boolean { + const parts = baseKey.split(':'); + if (parts.length === 1) { + return baseTranslatableColumns.includes(baseKey); + } + if (parts.length === 2) { + let customFieldConfigs: CustomFieldConfig[]; + if (parts[0] === 'product') { + customFieldConfigs = this.configService.customFields.Product; + } else if (parts[0] === 'variant') { + customFieldConfigs = this.configService.customFields.ProductVariant; + } else { + throw new InternalServerError(`Invalid column header '${baseKey}'`); + } + const customFieldConfig = customFieldConfigs.find(config => config.name === parts[1]); + if (!customFieldConfig) { + throw new InternalServerError( + `Could not find custom field config for column header '${baseKey}'`, + ); + } + return customFieldConfig.type === 'localeString'; + } + throw new InternalServerError(`Invalid column header '${baseKey}'`); + } + + private validateHeaderTranslations(rowKeys: string[]): string | undefined { + const missing: string[] = []; + const languageCodes = usedLanguageCodes(rowKeys); + const baseKeys = usedBaseKeys(rowKeys); + for (const baseKey of baseKeys) { + const translatedKeys = languageCodes.map(code => [baseKey, code].join(':')); + if (rowKeys.includes(baseKey)) { + // Untranslated column header is used -> there should be no translated ones + if (rowKeys.some(key => translatedKeys.includes(key))) { + return `The import file must not contain both translated and untranslated columns for field '${baseKey}'`; + } + } else { + if (!this.isTranslatable(baseKey) && translatedKeys.some(key => rowKeys.includes(key))) { + return `The '${baseKey}' field is not translatable.`; + } + // All column headers must exist for all translations + for (const translatedKey of translatedKeys) { + if (!rowKeys.includes(translatedKey)) { + missing.push(translatedKey); + } + } + } + } + if (missing.length) { + return `The import file is missing the following translations: ${missing + .map(m => `"${m}"`) + .join(', ')}`; + } + } + + private parseProductFromRecord( + r: { [key: string]: string }, + usedLanguages: LanguageCode[], + mainLanguage: LanguageCode, + ): ParsedProduct { + const translationCodes = usedLanguages.length === 0 ? [mainLanguage] : usedLanguages; + + const optionGroups: ParsedOptionGroup[] = []; + for (const languageCode of translationCodes) { + const rawTranslOptionGroups = r.hasOwnProperty(`optionGroups:${languageCode}`) + ? r[`optionGroups:${languageCode}`] + : r.optionGroups; + const translatedOptionGroups = parseStringArray(rawTranslOptionGroups); + if (optionGroups.length === 0) { + for (const translatedOptionGroup of translatedOptionGroups) { + optionGroups.push({ translations: [] }); + } + } + for (const i of optionGroups.map((optionGroup, index) => index)) { + optionGroups[i].translations.push({ + languageCode, + name: translatedOptionGroups[i], + values: [], + }); + } + } + + const facets: ParsedFacet[] = []; + for (const languageCode of translationCodes) { + const rawTranslatedFacets = r.hasOwnProperty(`facets:${languageCode}`) + ? r[`facets:${languageCode}`] + : r.facets; + const translatedFacets = parseStringArray(rawTranslatedFacets); + if (facets.length === 0) { + for (const translatedFacet of translatedFacets) { + facets.push({ translations: [] }); + } + } + for (const i of facets.map((facet, index) => index)) { + const [facet, value] = translatedFacets[i].split(':'); + facets[i].translations.push({ + languageCode, + facet, + value, + }); + } + } + + const translations = translationCodes.map(languageCode => { + const translatedFields = getRawTranslatedFields(r, languageCode); + const parsedTranslatedCustomFields = parseCustomFields('product', translatedFields); + const parsedUntranslatedCustomFields = parseCustomFields('product', getRawUntranslatedFields(r)); + const parsedCustomFields = { + ...parsedUntranslatedCustomFields, + ...parsedTranslatedCustomFields, + }; + const name = translatedFields.hasOwnProperty('name') + ? parseString(translatedFields.name) + : r.name; + let slug: string; + if (translatedFields.hasOwnProperty('slug')) { + slug = parseString(translatedFields.slug); + } else { + slug = parseString(r.slug); + } + if (slug.length === 0) { + slug = normalizeString(name, '-'); + } + return { + languageCode, + name, + slug, + description: translatedFields.hasOwnProperty('description') + ? parseString(translatedFields.description) + : r.description, + customFields: parsedCustomFields, + }; + }); + const parsedProduct: ParsedProduct = { + assetPaths: parseStringArray(r.assets), + optionGroups, + facets, + translations, + }; + return parsedProduct; + } + + private parseVariantFromRecord( + r: { [key: string]: string }, + usedLanguages: LanguageCode[], + mainLanguage: LanguageCode, + ): ParsedProductVariant { + const translationCodes = usedLanguages.length === 0 ? [mainLanguage] : usedLanguages; + + const facets: ParsedFacet[] = []; + for (const languageCode of translationCodes) { + const rawTranslatedFacets = r.hasOwnProperty(`variantFacets:${languageCode}`) + ? r[`variantFacets:${languageCode}`] + : r.variantFacets; + const translatedFacets = parseStringArray(rawTranslatedFacets); + if (facets.length === 0) { + for (const translatedFacet of translatedFacets) { + facets.push({ translations: [] }); + } + } + for (const i of facets.map((facet, index) => index)) { + const [facet, value] = translatedFacets[i].split(':'); + facets[i].translations.push({ + languageCode, + facet, + value, + }); + } + } + + const translations = translationCodes.map(languageCode => { + const rawTranslOptionValues = r.hasOwnProperty(`optionValues:${languageCode}`) + ? r[`optionValues:${languageCode}`] + : r.optionValues; + const translatedOptionValues = parseStringArray(rawTranslOptionValues); + const translatedFields = getRawTranslatedFields(r, languageCode); + const parsedTranslatedCustomFields = parseCustomFields('variant', translatedFields); + const parsedUntranslatedCustomFields = parseCustomFields('variant', getRawUntranslatedFields(r)); + const parsedCustomFields = { + ...parsedUntranslatedCustomFields, + ...parsedTranslatedCustomFields, + }; + return { + languageCode, + optionValues: translatedOptionValues, + customFields: parsedCustomFields, + }; + }); + + const parsedVariant: ParsedProductVariant = { + sku: parseString(r.sku), + price: parseNumber(r.price), + taxCategory: parseString(r.taxCategory), + stockOnHand: parseNumber(r.stockOnHand), + trackInventory: + r.trackInventory == null || r.trackInventory === '' + ? GlobalFlag.INHERIT + : parseBoolean(r.trackInventory) + ? GlobalFlag.TRUE + : GlobalFlag.FALSE, + assetPaths: parseStringArray(r.variantAssets), + facets, + translations, + }; + return parsedVariant; + } } function populateOptionGroupValues(currentRow: ParsedProductWithVariants) { - const values = currentRow.variants.map(v => v.optionValues); - currentRow.product.optionGroups.forEach((og, i) => { - og.values = unique(values.map(v => v[i])); - }); + for (const translation of currentRow.product.translations) { + const values = currentRow.variants.map(variant => { + const variantTranslation = variant.translations.find( + t => t.languageCode === translation.languageCode, + ); + if (!variantTranslation) { + throw new InternalServerError( + `No translation '${translation.languageCode}' for variant SKU '${variant.sku}'`, + ); + } + return variantTranslation.optionValues; + }); + currentRow.product.optionGroups.forEach((og, i) => { + const ogTranslation = og.translations.find(t => t.languageCode === translation.languageCode); + if (!ogTranslation) { + throw new InternalServerError(`No translation '${LanguageCode}' for option groups'`); + } + ogTranslation.values = unique(values.map(v => v[i])); + }); + } +} + +function getLanguageCode(rowKey: string): LanguageCode | undefined { + const parts = rowKey.split(':'); + if (parts.length === 2) { + if (parts[1] in LanguageCode) { + return parts[1] as LanguageCode; + } + } + if (parts.length === 3) { + if (['product', 'productVariant'].includes(parts[0]) && parts[2] in LanguageCode) { + return parts[2] as LanguageCode; + } + } +} + +function getBaseKey(rowKey: string): string { + const parts = rowKey.split(':'); + if (getLanguageCode(rowKey)) { + parts.pop(); + return parts.join(':'); + } else { + return rowKey; + } +} + +function usedLanguageCodes(rowKeys: string[]): LanguageCode[] { + const languageCodes: LanguageCode[] = []; + for (const rowKey of rowKeys) { + const languageCode = getLanguageCode(rowKey); + if (languageCode && !languageCodes.includes(languageCode)) { + languageCodes.push(languageCode); + } + } + return languageCodes; +} + +function usedBaseKeys(rowKeys: string[]): string[] { + const baseKeys: string[] = []; + for (const rowKey of rowKeys) { + const baseKey = getBaseKey(rowKey); + if (!baseKeys.includes(baseKey)) { + baseKeys.push(baseKey); + } + } + return baseKeys; } function validateRequiredColumns(r: string[]): string | undefined { const rowKeys = r; const missing: string[] = []; + const languageCodes = usedLanguageCodes(rowKeys); for (const col of requiredColumns) { if (!rowKeys.includes(col)) { + if (languageCodes.length > 0 && rowKeys.includes(`${col}:${languageCodes[0]}`)) { + continue; // If one translation is present, they are all present (we did 'validateHeaderTranslations' before) + } missing.push(col); } } @@ -213,70 +538,75 @@ function mapRowToObject(columns: string[], row: string[]): { [key: string]: stri } function validateOptionValueCount( - r: BaseProductRecord, + r: { [key: string]: string }, currentRow?: ParsedProductWithVariants, ): string | undefined { if (!currentRow) { return; } - const optionValues = parseStringArray(r.optionValues); - if (currentRow.product.optionGroups.length !== optionValues.length) { - return `The number of optionValues must match the number of optionGroups`; + + const optionValueKeys = Object.keys(r).filter(key => key.startsWith('optionValues')); + for (const key of optionValueKeys) { + const optionValues = parseStringArray(r[key]); + if (currentRow.product.optionGroups.length !== optionValues.length) { + return `The number of optionValues in column '${key}' must match the number of optionGroups`; + } + } +} + +function getRawMainTranslation( + r: { [key: string]: string }, + field: string, + mainLanguage: LanguageCode, +): string { + if (r.hasOwnProperty(field)) { + return r[field]; + } else { + return r[`${field}:${mainLanguage}`]; } } -function parseProductFromRecord(r: RawProductRecord): ParsedProduct { - const name = parseString(r.name); - const slug = parseString(r.slug) || normalizeString(name, '-'); - return { - name, - slug, - description: parseString(r.description), - assetPaths: parseStringArray(r.assets), - optionGroups: parseStringArray(r.optionGroups).map(ogName => ({ - name: ogName, - values: [], - })), - facets: parseStringArray(r.facets).map(pair => { - const [facet, value] = pair.split(':'); - return { facet, value }; - }), - customFields: parseCustomFields('product', r), - }; +function getRawTranslatedFields( + r: { [key: string]: string }, + languageCode: LanguageCode, +): { [key: string]: string } { + return Object.entries(r) + .filter(([key, value]) => key.endsWith(`:${languageCode}`)) + .reduce((output, [key, value]) => { + const fieldName = key.replace(`:${languageCode}`, ''); + return { + ...output, + [fieldName]: value, + }; + }, {}); } -function parseVariantFromRecord(r: RawProductRecord): ParsedProductVariant { - return { - optionValues: parseStringArray(r.optionValues), - sku: parseString(r.sku), - price: parseNumber(r.price), - taxCategory: parseString(r.taxCategory), - stockOnHand: parseNumber(r.stockOnHand), - trackInventory: - r.trackInventory == null || r.trackInventory === '' - ? GlobalFlag.INHERIT - : parseBoolean(r.trackInventory) - ? GlobalFlag.TRUE - : GlobalFlag.FALSE, - assetPaths: parseStringArray(r.variantAssets), - facets: parseStringArray(r.variantFacets).map(pair => { - const [facet, value] = pair.split(':'); - return { facet, value }; - }), - customFields: parseCustomFields('variant', r), - }; +function getRawUntranslatedFields(r: { [key: string]: string }): { [key: string]: string } { + return Object.entries(r) + .filter(([key, value]) => { + return !getLanguageCode(key); + }) + .reduce((output, [key, value]) => { + return { + ...output, + [key]: value, + }; + }, {}); } function isRelationObject(value: string) { try { const parsed = JSON.parse(value); return parsed && parsed.hasOwnProperty('id'); - } catch(e) { + } catch (e) { return false; } } -function parseCustomFields(prefix: 'product' | 'variant', r: RawProductRecord): { [name: string]: string } { +function parseCustomFields( + prefix: 'product' | 'variant', + r: { [key: string]: string }, +): { [name: string]: string } { return Object.entries(r) .filter(([key, value]) => { return key.indexOf(`${prefix}:`) === 0; diff --git a/packages/core/src/data-import/providers/import-parser/test-fixtures/multiple-languages.csv b/packages/core/src/data-import/providers/import-parser/test-fixtures/multiple-languages.csv new file mode 100644 index 0000000000..800101d792 --- /dev/null +++ b/packages/core/src/data-import/providers/import-parser/test-fixtures/multiple-languages.csv @@ -0,0 +1,4 @@ +name:en , name:zh_Hans , slug , description:en , description:zh_Hans , assets , facets:en , facets:zh_Hans , optionGroups , optionValues:en , optionValues:zh_Hans , sku , price , taxCategory , stockOnHand , trackInventory , variantAssets , variantFacets , product:keywords:en , product:keywords:zh_Hans , product:customPage , variant:volumetric +Perfect Paper Stretcher , 完美的纸张拉伸器 , , A great device for stretching paper. , 一个用于拉伸纸张的伟大装置 , , brand:KB|type:Accessory , 品牌:KB|类型:饰品 , size , Half Imperial , 半英制 , PPS12 , 45.3 , standard , 10 , false , , , "paper, stretch" , "纸张，拉伸" , grid-view , 243 + , , , , , , , , , Quarter Imperial , 四分之一英制 , PPS14 , 32.5 , standard , 10 , false , , , , , , 344 + , , , , , , , , , Full Imperial , 全英制 , PPSF , 59.5 , standard , 10 , false , , , , , , 656 diff --git a/packages/core/src/data-import/providers/importer/importer.ts b/packages/core/src/data-import/providers/importer/importer.ts index a14575f4ed..1c57529a8d 100644 --- a/packages/core/src/data-import/providers/importer/importer.ts +++ b/packages/core/src/data-import/providers/importer/importer.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { GlobalFlag, ImportInfo, LanguageCode } from '@vendure/common/lib/generated-types'; +import { ImportInfo, LanguageCode } from '@vendure/common/lib/generated-types'; import { normalizeString } from '@vendure/common/lib/normalize-string'; import { ID } from '@vendure/common/lib/shared-types'; import ProgressBar from 'progress'; @@ -7,6 +7,7 @@ import { Observable } from 'rxjs'; import { Stream } from 'stream'; import { RequestContext } from '../../../api/common/request-context'; +import { InternalServerError } from '../../../common/error/errors'; import { ConfigService } from '../../../config/config.service'; import { CustomFieldConfig } from '../../../config/custom-field/custom-field-types'; import { FacetValue } from '../../../entity/facet-value/facet-value.entity'; @@ -17,11 +18,7 @@ import { FacetValueService } from '../../../service/services/facet-value.service import { FacetService } from '../../../service/services/facet.service'; import { TaxCategoryService } from '../../../service/services/tax-category.service'; import { AssetImporter } from '../asset-importer/asset-importer'; -import { - ImportParser, - ParsedProductVariant, - ParsedProductWithVariants, -} from '../import-parser/import-parser'; +import { ImportParser, ParsedFacet, ParsedProductWithVariants } from '../import-parser/import-parser'; import { FastImporterService } from './fast-importer.service'; @@ -84,7 +81,7 @@ export class Importer { onProgress: OnProgressFn, ): Promise { const ctx = await this.getRequestContext(ctxOrLanguageCode); - const parsed = await this.importParser.parseProducts(input); + const parsed = await this.importParser.parseProducts(input, ctx.languageCode); if (parsed && parsed.results.length) { try { const importErrors = await this.importProducts(ctx, parsed.results, progess => { @@ -145,61 +142,79 @@ export class Importer { const taxCategories = await this.taxCategoryService.findAll(ctx); await this.fastImporter.initialize(); for (const { product, variants } of rows) { + const productMainTranslation = this.getTranslationByCodeOrFirst( + product.translations, + ctx.languageCode, + ); const createProductAssets = await this.assetImporter.getAssets(product.assetPaths); const productAssets = createProductAssets.assets; if (createProductAssets.errors.length) { errors = errors.concat(createProductAssets.errors); } const customFields = this.processCustomFieldValues( - product.customFields, + product.translations[0].customFields, this.configService.customFields.Product, ); const createdProductId = await this.fastImporter.createProduct({ featuredAssetId: productAssets.length ? productAssets[0].id : undefined, assetIds: productAssets.map(a => a.id), facetValueIds: await this.getFacetValueIds(product.facets, languageCode), - translations: [ - { - languageCode, - name: product.name, - description: product.description, - slug: product.slug, - customFields, - }, - ], + translations: product.translations.map(translation => { + return { + languageCode: translation.languageCode, + name: translation.name, + description: translation.description, + slug: translation.slug, + customFields: this.processCustomFieldValues( + translation.customFields, + this.configService.customFields.Product, + ), + }; + }), customFields, }); const optionsMap: { [optionName: string]: ID } = {}; for (const optionGroup of product.optionGroups) { - const code = normalizeString(`${product.name}-${optionGroup.name}`, '-'); + const optionGroupMainTranslation = this.getTranslationByCodeOrFirst( + optionGroup.translations, + ctx.languageCode, + ); + const code = normalizeString( + `${productMainTranslation.name}-${optionGroupMainTranslation.name}`, + '-', + ); const groupId = await this.fastImporter.createProductOptionGroup({ code, - options: optionGroup.values.map(name => ({} as any)), - translations: [ - { - languageCode, - name: optionGroup.name, - }, - ], + options: optionGroupMainTranslation.values.map(name => ({} as any)), + translations: optionGroup.translations.map(translation => { + return { + languageCode: translation.languageCode, + name: translation.name, + }; + }), }); - for (const option of optionGroup.values) { + for (const optionIndex of optionGroupMainTranslation.values.map((value, index) => index)) { const createdOptionId = await this.fastImporter.createProductOption({ productOptionGroupId: groupId, - code: normalizeString(option, '-'), - translations: [ - { - languageCode, - name: option, - }, - ], + code: normalizeString(optionGroupMainTranslation.values[optionIndex], '-'), + translations: optionGroup.translations.map(translation => { + return { + languageCode: translation.languageCode, + name: translation.values[optionIndex], + }; + }), }); - optionsMap[option] = createdOptionId; + optionsMap[optionGroupMainTranslation.values[optionIndex]] = createdOptionId; } await this.fastImporter.addOptionGroupToProduct(createdProductId, groupId); } for (const variant of variants) { + const variantMainTranslation = this.getTranslationByCodeOrFirst( + variant.translations, + ctx.languageCode, + ); const createVariantAssets = await this.assetImporter.getAssets(variant.assetPaths); const variantAssets = createVariantAssets.assets; if (createVariantAssets.errors.length) { @@ -209,6 +224,10 @@ export class Importer { if (0 < variant.facets.length) { facetValueIds = await this.getFacetValueIds(variant.facets, languageCode); } + const variantCustomFields = this.processCustomFieldValues( + variantMainTranslation.customFields, + this.configService.customFields.ProductVariant, + ); const createdVariant = await this.fastImporter.createProductVariant({ productId: createdProductId, facetValueIds, @@ -218,18 +237,27 @@ export class Importer { taxCategoryId: this.getMatchingTaxCategoryId(variant.taxCategory, taxCategories), stockOnHand: variant.stockOnHand, trackInventory: variant.trackInventory, - optionIds: variant.optionValues.map(v => optionsMap[v]), - translations: [ - { - languageCode, - name: [product.name, ...variant.optionValues].join(' '), - }, - ], + optionIds: variantMainTranslation.optionValues.map(v => optionsMap[v]), + translations: variant.translations.map(translation => { + const productTranslation = product.translations.find( + t => t.languageCode === translation.languageCode, + ); + if (!productTranslation) { + throw new InternalServerError( + `No translation '${translation.languageCode}' for product with slug '${productMainTranslation.slug}'`, + ); + } + return { + languageCode: translation.languageCode, + name: [productTranslation.name, ...translation.optionValues].join(' '), + customFields: this.processCustomFieldValues( + translation.customFields, + this.configService.customFields.ProductVariant, + ), + }; + }), price: Math.round(variant.price * 100), - customFields: this.processCustomFieldValues( - variant.customFields, - this.configService.customFields.ProductVariant, - ), + customFields: variantCustomFields, }); } imported++; @@ -237,16 +265,13 @@ export class Importer { processed: 0, imported, errors, - currentProduct: product.name, + currentProduct: productMainTranslation.name, }); } return errors; } - private async getFacetValueIds( - facets: ParsedProductVariant['facets'], - languageCode: LanguageCode, - ): Promise { + private async getFacetValueIds(facets: ParsedFacet[], languageCode: LanguageCode): Promise { const facetValueIds: ID[] = []; const ctx = new RequestContext({ channel: await this.channelService.getDefaultChannel(), @@ -257,8 +282,9 @@ export class Importer { }); for (const item of facets) { - const facetName = item.facet; - const valueName = item.value; + const itemMainTranslation = this.getTranslationByCodeOrFirst(item.translations, languageCode); + const facetName = itemMainTranslation.facet; + const valueName = itemMainTranslation.value; let facetEntity: Facet; const cachedFacet = this.facetMap.get(facetName); @@ -272,7 +298,12 @@ export class Importer { facetEntity = await this.facetService.create(ctx, { isPrivate: false, code: normalizeString(facetName, '-'), - translations: [{ languageCode, name: facetName }], + translations: item.translations.map(translation => { + return { + languageCode: translation.languageCode, + name: translation.facet, + }; + }), }); } this.facetMap.set(facetName, facetEntity); @@ -293,7 +324,12 @@ export class Importer { facetEntity, { code: normalizeString(valueName, '-'), - translations: [{ languageCode, name: valueName }], + translations: item.translations.map(translation => { + return { + languageCode: translation.languageCode, + name: translation.value, + }; + }), }, ); } @@ -329,4 +365,15 @@ export class Importer { this.taxCategoryMatches[name] = match.id; return match.id; } + + private getTranslationByCodeOrFirst( + translations: Type[], + languageCode: LanguageCode, + ): Type { + let translation = translations.find(t => t.languageCode === languageCode); + if (!translation) { + translation = translations[0]; + } + return translation; + } }