From e9df5b2ae1f74d27a976ca18f374b730e8928419 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Tue, 27 Aug 2024 16:55:10 +0800 Subject: [PATCH 01/11] feat(core): looser name length --- apps/cli/src/linter/schema.ts | 5 ++++- apps/cli/src/linter/specs/common.spec.ts | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/apps/cli/src/linter/schema.ts b/apps/cli/src/linter/schema.ts index 3b268a2..dbd59eb 100644 --- a/apps/cli/src/linter/schema.ts +++ b/apps/cli/src/linter/schema.ts @@ -1,6 +1,9 @@ import { z } from 'zod'; -const nameSchema = z.string().min(1).max(100); +const nameSchema = z + .string() + .min(1) + .max(64 * 1024); const descriptionSchema = z .string() .max(64 * 1024) diff --git a/apps/cli/src/linter/specs/common.spec.ts b/apps/cli/src/linter/specs/common.spec.ts index d12d784..c3fcdd1 100644 --- a/apps/cli/src/linter/specs/common.spec.ts +++ b/apps/cli/src/linter/specs/common.spec.ts @@ -5,11 +5,11 @@ import { check } from '../'; describe('Common Linter', () => { const cases = [ { - name: 'should check description length (length <= 64 * 1024)', + name: 'should check name/description length (length <= 64 * 1024)', input: { services: [ { - name: 'test', + name: ''.padEnd(64 * 1024, '0'), description: ''.padEnd(64 * 1024, '0'), routes: [], }, @@ -19,11 +19,11 @@ describe('Common Linter', () => { errors: [], }, { - name: 'should check description length (length > 64 * 1024)', + name: 'should check name/description length (length > 64 * 1024)', input: { services: [ { - name: 'test', + name: ''.padEnd(64 * 1024 + 1, '0'), description: ''.padEnd(64 * 1024 + 1, '0'), routes: [], }, @@ -31,6 +31,15 @@ describe('Common Linter', () => { } as ADCSDK.Configuration, expect: false, errors: [ + { + code: 'too_big', + exact: false, + inclusive: true, + maximum: 65536, + message: 'String must contain at most 65536 character(s)', + path: ['services', 0, 'name'], + type: 'string', + }, { code: 'too_big', exact: false, From 44a72818a9dac256b33ccb8e40f1d9c66282f43f Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Tue, 27 Aug 2024 18:20:32 +0800 Subject: [PATCH 02/11] chore: update schema.json --- schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.json b/schema.json index 295f3ac..b9afd10 100644 --- a/schema.json +++ b/schema.json @@ -9,7 +9,7 @@ "name": { "type": "string", "minLength": 1, - "maxLength": 100 + "maxLength": 65536 }, "description": { "type": "string", From 39e932bc9c97c9d9db3a3bbf8aeb66bde1fcbeb4 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Thu, 29 Aug 2024 15:34:25 +0800 Subject: [PATCH 03/11] test: name length e2e --- .github/workflows/e2e.yaml | 2 +- libs/backend-api7/e2e/misc.e2e-spec.ts | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6d688a6..f55fc22 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -39,7 +39,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'test/api7') || github.event_name == 'push' strategy: matrix: - version: [3.2.13.0, 3.2.14.5] + version: [3.2.13.0, 3.2.14.6] env: BACKEND_API7_DOWNLOAD_URL: https://run.api7.ai/api7-ee/api7-ee-v${{ matrix.version }}.tar.gz BACKEND_API7_LICENSE: ${{ secrets.BACKEND_API7_LICENSE }} diff --git a/libs/backend-api7/e2e/misc.e2e-spec.ts b/libs/backend-api7/e2e/misc.e2e-spec.ts index 877deec..c95919c 100644 --- a/libs/backend-api7/e2e/misc.e2e-spec.ts +++ b/libs/backend-api7/e2e/misc.e2e-spec.ts @@ -24,8 +24,13 @@ describe('Miscellaneous', () => { }); }); - describe('Sync resources with the description greater than 256 bytes', () => { - const service1Name = 'service1'; + describe('Sync resources with the name/description greater than 256 bytes', () => { + const routeName = ''.padEnd(64 * 1024, '0'); // 65536 bytes + const service1Name = ''.padEnd(64 * 1024, '0'); // 65536 bytes + const route = { + name: routeName, + uris: ['/test'], + }; const service1 = { name: service1Name, description: ''.padEnd(64 * 1024, '0'), // 65536 bytes @@ -39,6 +44,7 @@ describe('Miscellaneous', () => { }, ], }, + routes: [route], } as ADCSDK.Service; it('Create services', async () => @@ -50,6 +56,8 @@ describe('Miscellaneous', () => { const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; expect(result.services).toHaveLength(1); expect(result.services[0]).toMatchObject(service1); + expect(result.services[0]).toMatchObject(service1); + expect(result.services[0].routes[0]).toMatchObject(route); }); it('Delete service', async () => From 4681ddfcb2b39fcee861a415080289e4514422af Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 13:01:28 +0800 Subject: [PATCH 04/11] feat: add stream service internal type --- libs/sdk/src/core/resource.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/sdk/src/core/resource.ts b/libs/sdk/src/core/resource.ts index 646b4d1..445703a 100644 --- a/libs/sdk/src/core/resource.ts +++ b/libs/sdk/src/core/resource.ts @@ -9,4 +9,7 @@ export enum ResourceType { CONSUMER = 'consumer', CONSUMER_GROUP = 'consumer_group', STREAM_ROUTE = 'stream_route', + + // internal use only + INTERNAL_STREAM_SERVICE = 'stream_service', } From 98b5b092d331be8a36056ef580e3f3944ee8b9f6 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 13:07:04 +0800 Subject: [PATCH 05/11] feat: extend differ v3 for new stream service type --- apps/cli/src/differ/differv3.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/differ/differv3.ts b/apps/cli/src/differ/differv3.ts index 815b23e..f838a5a 100644 --- a/apps/cli/src/differ/differv3.ts +++ b/apps/cli/src/differ/differv3.ts @@ -1,7 +1,7 @@ import * as ADCSDK from '@api7/adc-sdk'; import { randomUUID } from 'crypto'; import { diff as objectDiff } from 'deep-diff'; -import { cloneDeep, isEmpty, isEqual, isNil, unset } from 'lodash'; +import { cloneDeep, has, isEmpty, isEqual, isNil, unset } from 'lodash'; import winston from 'winston'; const order = { @@ -359,6 +359,20 @@ export class DifferV3 { ); } + // Services will have two different default value rules depending on + // the type of route they contain, one for HTTP services and another + // for stream services. + // Therefore, before merging the default values, we should decide + // the default value table according to the type of service. This type + // is only used for merging default values, other processes still + // use the ResourceType.SERVICE type. + const resourceTypeForDefault = + resourceType != ADCSDK.ResourceType.SERVICE + ? resourceType + : has(localItem, 'stream_routes') + ? ADCSDK.ResourceType.INTERNAL_STREAM_SERVICE + : ADCSDK.ResourceType.SERVICE; + // Merges local resources into a table of default values for this type. // The Admin API merges the default values specified in the schema into // the data, so default values not contained in the local data will @@ -368,7 +382,8 @@ export class DifferV3 { // As such, each backend implementation needs to provide its table of // default values, which merges the local resources to the defaults, // just as the Admin API does. - const defaultValue = this.defaultValue?.core?.[resourceType] ?? {}; + const defaultValue = + this.defaultValue?.core?.[resourceTypeForDefault] ?? {}; const mergedLocalItem = this.mergeDefault( localItem, cloneDeep(defaultValue), From 8443ac19521eac3ae64345a190fc405c7176fbfa Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 13:20:02 +0800 Subject: [PATCH 06/11] feat: add stream service type for backend --- libs/backend-api7/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/backend-api7/src/index.ts b/libs/backend-api7/src/index.ts index aa49fea..fcdf8e1 100644 --- a/libs/backend-api7/src/index.ts +++ b/libs/backend-api7/src/index.ts @@ -122,6 +122,7 @@ export class BackendAPI7 implements ADCSDK.Backend { switch (type) { case ADCSDK.ResourceType.ROUTE: return toADC.transformRoute; + case ADCSDK.ResourceType.INTERNAL_STREAM_SERVICE: case ADCSDK.ResourceType.SERVICE: return toADC.transformService; case ADCSDK.ResourceType.SSL: From ef5687cd287271a404bac086f0ec230b6381c018 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 14:48:32 +0800 Subject: [PATCH 07/11] test: fix --- libs/backend-api7/e2e/misc.e2e-spec.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libs/backend-api7/e2e/misc.e2e-spec.ts b/libs/backend-api7/e2e/misc.e2e-spec.ts index c95919c..00a049b 100644 --- a/libs/backend-api7/e2e/misc.e2e-spec.ts +++ b/libs/backend-api7/e2e/misc.e2e-spec.ts @@ -26,13 +26,13 @@ describe('Miscellaneous', () => { describe('Sync resources with the name/description greater than 256 bytes', () => { const routeName = ''.padEnd(64 * 1024, '0'); // 65536 bytes - const service1Name = ''.padEnd(64 * 1024, '0'); // 65536 bytes + const serviceName = ''.padEnd(64 * 1024, '0'); // 65536 bytes const route = { name: routeName, uris: ['/test'], }; - const service1 = { - name: service1Name, + const service = { + name: serviceName, description: ''.padEnd(64 * 1024, '0'), // 65536 bytes upstream: { scheme: 'https', @@ -49,20 +49,21 @@ describe('Miscellaneous', () => { it('Create services', async () => syncEvents(backend, [ - createEvent(ADCSDK.ResourceType.SERVICE, service1Name, service1), + createEvent(ADCSDK.ResourceType.SERVICE, serviceName, service), + createEvent(ADCSDK.ResourceType.ROUTE, routeName, route), ])); it('Dump', async () => { const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; expect(result.services).toHaveLength(1); - expect(result.services[0]).toMatchObject(service1); - expect(result.services[0]).toMatchObject(service1); + expect(result.services[0]).toMatchObject(service); + expect(result.services[0]).toMatchObject(service); expect(result.services[0].routes[0]).toMatchObject(route); }); it('Delete service', async () => syncEvents(backend, [ - deleteEvent(ADCSDK.ResourceType.SERVICE, service1Name), + deleteEvent(ADCSDK.ResourceType.SERVICE, serviceName), ])); it('Dump again', async () => { From c83d0171ed547e1a2025ae84468cea5a14483aec Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 15:18:39 +0800 Subject: [PATCH 08/11] test: fix --- libs/backend-api7/e2e/misc.e2e-spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/backend-api7/e2e/misc.e2e-spec.ts b/libs/backend-api7/e2e/misc.e2e-spec.ts index 00a049b..49da84e 100644 --- a/libs/backend-api7/e2e/misc.e2e-spec.ts +++ b/libs/backend-api7/e2e/misc.e2e-spec.ts @@ -50,7 +50,7 @@ describe('Miscellaneous', () => { it('Create services', async () => syncEvents(backend, [ createEvent(ADCSDK.ResourceType.SERVICE, serviceName, service), - createEvent(ADCSDK.ResourceType.ROUTE, routeName, route), + createEvent(ADCSDK.ResourceType.ROUTE, routeName, route, serviceName), ])); it('Dump', async () => { @@ -63,6 +63,7 @@ describe('Miscellaneous', () => { it('Delete service', async () => syncEvents(backend, [ + deleteEvent(ADCSDK.ResourceType.ROUTE, routeName, serviceName), deleteEvent(ADCSDK.ResourceType.SERVICE, serviceName), ])); From 67730923cec4ca964ad299a8d4dc7365cec7ff55 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 15:23:06 +0800 Subject: [PATCH 09/11] test: remove 3.2.13.0 support --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f55fc22..6652f50 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -39,7 +39,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'test/api7') || github.event_name == 'push' strategy: matrix: - version: [3.2.13.0, 3.2.14.6] + version: [3.2.14.6] env: BACKEND_API7_DOWNLOAD_URL: https://run.api7.ai/api7-ee/api7-ee-v${{ matrix.version }}.tar.gz BACKEND_API7_LICENSE: ${{ secrets.BACKEND_API7_LICENSE }} From 8725058caa55fb53f8a8b3f762be1debf101a0f8 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 15:25:24 +0800 Subject: [PATCH 10/11] chore: clean --- libs/backend-api7/e2e/misc.e2e-spec.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/backend-api7/e2e/misc.e2e-spec.ts b/libs/backend-api7/e2e/misc.e2e-spec.ts index 49da84e..0bdcca7 100644 --- a/libs/backend-api7/e2e/misc.e2e-spec.ts +++ b/libs/backend-api7/e2e/misc.e2e-spec.ts @@ -1,7 +1,4 @@ import * as ADCSDK from '@api7/adc-sdk'; -import { unset } from 'lodash'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; import { BackendAPI7 } from '../src'; import { @@ -9,7 +6,6 @@ import { deleteEvent, dumpConfiguration, syncEvents, - updateEvent, } from './support/utils'; describe('Miscellaneous', () => { From c6293313acf7891d39a4331da73a302d0c4fb209 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 30 Aug 2024 15:34:32 +0800 Subject: [PATCH 11/11] fix: apisix ignore stream service type --- libs/backend-apisix/src/fetcher.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/backend-apisix/src/fetcher.ts b/libs/backend-apisix/src/fetcher.ts index 3edadc9..73f67ff 100644 --- a/libs/backend-apisix/src/fetcher.ts +++ b/libs/backend-apisix/src/fetcher.ts @@ -32,6 +32,10 @@ export class Fetcher { private allTask(): Array { return Object.values(ADCSDK.ResourceType) + .filter( + (resourceType) => + resourceType !== ADCSDK.ResourceType.INTERNAL_STREAM_SERVICE, // ignore internal only types + ) .map((resourceType): [ADCSDK.ResourceType, string] => [ resourceType, resourceTypeToAPIName(resourceType),