diff --git a/SPECIFICATION_TESTING.md b/SPECIFICATION_TESTING.md index 65386f57c..1c63d2276 100644 --- a/SPECIFICATION_TESTING.md +++ b/SPECIFICATION_TESTING.md @@ -138,7 +138,7 @@ As you can see, the `output` field in the first chapter saves the `model_group_i ### Managing Versions -It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your tests as follows. +It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows. ```yaml - synopsis: Search with `phase_took` added in OpenSearch 2.12. diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index cff3111fa..e2663a159 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -18,6 +18,8 @@ properties: type: array items: $ref: '#/definitions/Chapter' + version: + $ref: '#/definitions/Version' required: [description, chapters] additionalProperties: false diff --git a/tests/cat/pit_segments.yaml b/tests/cat/pit_segments.yaml index 478a39ef8..ab8dd5f87 100644 --- a/tests/cat/pit_segments.yaml +++ b/tests/cat/pit_segments.yaml @@ -1,6 +1,7 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test cat/pit_segments endpoints. +version: '>= 2.4.0' epilogues: - path: /games method: DELETE diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 2a5998a2e..db751e543 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -26,6 +26,16 @@ export default class StoryEvaluator { } async evaluate({ story, display_path, full_path }: StoryFile, version: string, dry_run: boolean = false): Promise { + if (story.version !== undefined && !semver.satisfies(version, story.version)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because version ${version} does not satisfy ${story.version}.` + } + } + const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error @@ -53,7 +63,7 @@ export default class StoryEvaluator { evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else if (chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because ${version} does not satisfy ${chapter.version}.`, error: undefined } }) + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 7c5b1d0df..4581a617e 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -65,11 +65,12 @@ export type ReadChapter = Chapter & { }; export interface Story { - $schema: string; + $schema?: string; description: string; prologues?: SupplementalChapter[]; epilogues?: SupplementalChapter[]; chapters: Chapter[]; + version?: Version; } /** * This interface was referenced by `Story`'s JSON-Schema diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index 3b4c9ca23..6808a9da1 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -136,7 +136,7 @@ chapters: - title: This GET /_cat/health should be skipped (> 2.999.0). overall: result: SKIPPED - message: Skipped because 2.15.0 does not satisfy >= 2.999.0. + message: Skipped because version 2.15.0 does not satisfy >= 2.999.0. epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/evals/skipped/semver.yaml b/tools/tests/tester/fixtures/evals/skipped/semver.yaml new file mode 100644 index 000000000..c6ae7a4de --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/semver.yaml @@ -0,0 +1,6 @@ +display_path: skipped/semver.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/semver.yaml + +result: SKIPPED +description: This story should be skipped because of version. +message: Skipped because version 2.15.0 does not satisfy >= 2.999.0. diff --git a/tools/tests/tester/fixtures/stories/skipped/semver.yaml b/tools/tests/tester/fixtures/stories/skipped/semver.yaml new file mode 100644 index 000000000..11defd91e --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/semver.yaml @@ -0,0 +1,7 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of version. +version: '>= 2.999.0' +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index 250a1abfe..a72e6a5b0 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -29,11 +29,12 @@ test('stories folder', async () => { } const passed = load_expected_evaluation('passed', true) + const skipped = load_expected_evaluation('skipped/semver', true) const not_found = load_expected_evaluation('failed/not_found', true) const invalid_data = load_expected_evaluation('failed/invalid_data', true) const chapter_error = load_expected_evaluation('error/chapter_error', true) const prologue_error = load_expected_evaluation('error/prologue_error', true) - const expected_evaluations = [passed, chapter_error, prologue_error, invalid_data, not_found] + const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] expect(actual_evaluations).toEqual(expected_evaluations) })