Run integration tests against OpenSearch 2.16.

.cspell

@@ -125,6 +125,7 @@ nysiis
 opendistro
 opensearch
 opensearchproject
+opensearchstaging
 ords
 oversample
 performanceanalyzer

.github/opensearch-cluster/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   opensearch-cluster:
-    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-latest}
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}
     ports:
       - "9200:9200"
       - "9600:9600"

.github/workflows/test-spec.yml

@@ -2,25 +2,32 @@ name: Test Spec
 
 on:
   push:
-    branches: ['**']
     paths:
-      - 'package*.json'
+      - .github/workflows/test-spec.yml
+      - package*.json
       - tsconfig.json
-      - 'tools/src/tester/**'
-      - 'spec/**'
+      - tools/src/tester/**
+      - spec/**
   pull_request:
-    branches: ['**']
     paths:
-      - 'package*.json'
+      - .github/workflows/test-spec.yml
+      - package*.json
       - tsconfig.json
-      - 'tools/src/tester/**'
-      - 'spec/**'
+      - tools/src/tester/**
+      - spec/**
 
 jobs:
   test-opensearch-spec:
+    strategy:
+      matrix:
+        entry:
+          - {version: 2.15.0, hub: 'opensearchproject'}
+          - {version: 2.16.0, hub: 'opensearchstaging'}
+    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub }})
     runs-on: ubuntu-latest
     env:
-      OPENSEARCH_VERSION: 2.15.0
+      OPENSEARCH_DOCKER_HUB_PROJECT: ${{ matrix.entry.hub }}
+      OPENSEARCH_VERSION: ${{ matrix.entry.version }}
       OPENSEARCH_PASSWORD: myStrongPassword123!
     steps:
       - name: Checkout Repo
@@ -36,7 +43,7 @@ jobs:
 
       - name: Run OpenSearch Cluster
         working-directory: .github/opensearch-cluster
-        run: docker-compose up -d && sleep 60
+        run: docker-compose up -d
 
       - name: Run Tests
         run: npm run test:spec -- --opensearch-insecure

CHANGELOG.md

@@ -40,6 +40,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `concurrent_query_*` and `search_idle_reactivate_count_total` fields to `SearchStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `remote_store` to `TranslogStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `file` to `/_cache/clear` and `/{index}/_cache/clear` ([#396](https://github.com/opensearch-project/opensearch-api-specification/pull/396))
+- Added a workflow to run tests against the next version of OpenSearch ([#409](https://github.com/opensearch-project/opensearch-api-specification/pull/409))
 
 ### Changed
 

tools/src/OpenSearchHttpClient.ts

@@ -71,8 +71,10 @@ export interface OpenSearchInfo {
 
 export class OpenSearchHttpClient {
   private readonly _axios: AxiosInstance
+  private readonly _opts?: OpenSearchHttpClientOptions
 
   constructor (opts?: OpenSearchHttpClientOptions) {
+    this._opts = opts
     this._axios = axios.create({
       baseURL: opts?.url ?? DEFAULT_URL,
       auth: opts?.username !== undefined && opts.password !== undefined
@@ -92,8 +94,17 @@ export class OpenSearchHttpClient {
       attempt += 1
       try {
         const info = await this.get('/')
-        return info.data
+        if (this._opts?.responseType == 'arraybuffer') {
+          return JSON.parse(info.data as string)
+        } else {
+          return info.data
+        }
       } catch (e) {
+        if (axios.isAxiosError(e)) {
+          if (e.code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
+            throw e
+          }
+        }
         if (attempt >= max_attempts) {
           throw e
         }

tools/src/tester/TestRunner.ts

@@ -16,13 +16,17 @@ import { Result, type StoryEvaluation } from './types/eval.types'
 import { type ResultLogger } from './ResultLogger'
 import { basename, resolve } from 'path'
 import type StoryValidator from "./StoryValidator";
+import { OpenSearchHttpClient } from 'OpenSearchHttpClient'
+import * as ansi from './Ansi'
 
 export default class TestRunner {
+  private readonly _http_client: OpenSearchHttpClient
   private readonly _story_validator: StoryValidator
   private readonly _story_evaluator: StoryEvaluator
   private readonly _result_logger: ResultLogger
 
-  constructor (story_validator: StoryValidator, story_evaluator: StoryEvaluator, result_logger: ResultLogger) {
+  constructor (http_client: OpenSearchHttpClient, story_validator: StoryValidator, story_evaluator: StoryEvaluator, result_logger: ResultLogger) {
+    this._http_client = http_client
     this._story_validator = story_validator
     this._story_evaluator = story_evaluator
     this._result_logger = result_logger
@@ -32,6 +36,12 @@ export default class TestRunner {
     let failed = false
     const story_files = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', ''))
     const evaluations: StoryEvaluation[] = []
+
+    if (!dry_run) {
+      const info = await this._http_client.wait_until_available()
+      console.log(`OpenSearch ${ansi.green(info.version.number)}\n`)
+    }
+
     for (const story_file of story_files) {
       const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, dry_run)
       evaluations.push(evaluation)

tools/src/tester/test.ts

@@ -58,7 +58,7 @@ const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_
 const story_validator = new StoryValidator()
 const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
 const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose)
-const runner = new TestRunner(story_validator, story_evaluator, result_logger)
+const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger)
 
 runner.run(opts.testsPath, opts.dryRun)
   .then(

tools/tests/tester/helpers.ts

@@ -52,7 +52,7 @@ export function construct_tester_components (spec_path: string): {
   const story_validator = new StoryValidator()
   const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
   const result_logger = new NoOpResultLogger()
-  const test_runner = new TestRunner(story_validator, story_evaluator, result_logger)
+  const test_runner = new TestRunner(opensearch_http_client, story_validator, story_evaluator, result_logger)
   return {
     specification,
     operation_locator,

tools/tests/tester/test.test.ts

@@ -34,13 +34,13 @@ test('--invalid', () => {
 })
 
 test('displays story filename', () => {
-  expect(spec(['--tests', 'tools/tests/tester/fixtures/empty_story']).stdout).toContain(
+  expect(spec(['--dry-run', '--tests', 'tools/tests/tester/fixtures/empty_story']).stdout).toContain(
     `${ansi.green('PASSED ')} ${ansi.cyan(ansi.b('empty.yaml'))}`
   )
 })
 
 test('invalid story', () => {
-  expect(spec(['--tests', 'tools/tests/tester/fixtures/invalid_story.yaml']).stdout).toContain(
+  expect(spec(['--dry-run', '--tests', 'tools/tests/tester/fixtures/invalid_story.yaml']).stdout).toContain(
     `\x1b[90m(Invalid Story:`
   )
 })