Enable SPEC SAT for Java sources (#18779)

* enable py spec compatibility tests

* add missing properties

* use expected spec file instead of the source spec.json + use dummy config file

* add missing files for the clickhouse

* move test files to the integration-test from the unit test folder

* add missing files to cockroachdb

* add missing files to db2 + fix spec format

* add missing files to elasticsearch + fix spec format

* add missing files to jdbc

* add missing files to mongodb_v2 + fix spec format

* add missing files to mssql + fix spec format

* add missing files to mysql + fix spec format

* add missing files to postgres + fix spec format

* add missing files to oracle + fix spec format

* add missing files to redshift

* add missing files to sftp

* add missing files to snowflake + fix spec format

* add missing files to tidb

* add missing files to kafka - fix spec format

* airbyte-source-acceptance-test added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* add missing import

* Delete acceptance-test-config.yml

* Delete acceptance-test-docker.sh

* Update build.gradle

* Update build.gradle

* format

* revert changes

* manual .sh files format

* upd expected spec

* format

* fix SAT after master merge

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
Co-authored-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml

@@ -0,0 +1,7 @@
+# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-bigquery:dev
+tests:
+  spec:
+    - spec_path: "src/test-integration/resources/expected_spec.json"
+      config_path: "src/test-integration/resources/dummy_config.json"

airbyte-integrations/connectors/source-bigquery/acceptance-test-docker.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v /tmp:/tmp \
+  -v $(pwd):/test_input \
+  airbyte/source-acceptance-test \
+  --acceptance-test-config /test_input

airbyte-integrations/connectors/source-bigquery/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id 'application'
     id 'airbyte-docker'
     id 'airbyte-integration-test-java'
+    id 'airbyte-source-acceptance-test'
 }
 
 application {

airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
+    yield
+    # TODO: clean up test dependencies

airbyte-integrations/connectors/source-bigquery/src/test-integration/resources/dummy_config.json

@@ -0,0 +1,5 @@
+{
+  "dataset_id": "dataset",
+  "project_id": "project",
+  "credentials_json": "credentials"
+}

airbyte-integrations/connectors/source-bigquery/src/test-integration/resources/expected_spec.json

@@ -0,0 +1,32 @@
+{
+  "documentationUrl": "https://docs.airbyte.com/integrations/sources/bigquery",
+  "supportsIncremental": true,
+  "supportsNormalization": true,
+  "supportsDBT": true,
+  "supported_destination_sync_modes": [],
+  "supported_sync_modes": ["overwrite", "append", "append_dedup"],
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "BigQuery Source Spec",
+    "type": "object",
+    "required": ["project_id", "credentials_json"],
+    "properties": {
+      "project_id": {
+        "type": "string",
+        "description": "The GCP project ID for the project containing the target BigQuery dataset.",
+        "title": "Project ID"
+      },
+      "dataset_id": {
+        "type": "string",
+        "description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery.",
+        "title": "Default Dataset ID"
+      },
+      "credentials_json": {
+        "type": "string",
+        "description": "The contents of your Service Account Key JSON file. See the <a href=\"https://docs.airbyte.com/integrations/sources/bigquery#setup-the-bigquery-source-in-airbyte\">docs</a> for more information on how to obtain this key.",
+        "title": "Credentials JSON",
+        "airbyte_secret": true
+      }
+    }
+  }
+}

airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml

@@ -0,0 +1,7 @@
+# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-clickhouse:dev
+tests:
+  spec:
+    - spec_path: "src/test-integration/resources/expected_spec.json"
+      config_path: "src/test-integration/resources/dummy_config.json"

airbyte-integrations/connectors/source-clickhouse/acceptance-test-docker.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v /tmp:/tmp \
+  -v $(pwd):/test_input \
+  airbyte/source-acceptance-test \
+  --acceptance-test-config /test_input

airbyte-integrations/connectors/source-clickhouse/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id 'application'
     id 'airbyte-docker'
     id 'airbyte-integration-test-java'
+    id 'airbyte-source-acceptance-test'
 }
 
 application {

airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
+    yield
+    # TODO: clean up test dependencies

airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/dummy_config.json

@@ -0,0 +1,6 @@
+{
+  "host": "default",
+  "port": 8123,
+  "database": "default",
+  "username": "default"
+}

airbyte-integrations/connectors/source-clickhouse/src/test-integration/resources/expected_spec.json

@@ -0,0 +1,177 @@
+{
+  "documentationUrl": "https://docs.airbyte.com/integrations/destinations/clickhouse",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "ClickHouse Source Spec",
+    "type": "object",
+    "required": ["host", "port", "database", "username"],
+    "properties": {
+      "host": {
+        "description": "The host endpoint of the Clickhouse cluster.",
+        "title": "Host",
+        "type": "string",
+        "order": 0
+      },
+      "port": {
+        "description": "The port of the database.",
+        "title": "Port",
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 65536,
+        "default": 8123,
+        "examples": ["8123"],
+        "order": 1
+      },
+      "database": {
+        "description": "The name of the database.",
+        "title": "Database",
+        "type": "string",
+        "examples": ["default"],
+        "order": 2
+      },
+      "username": {
+        "description": "The username which is used to access the database.",
+        "title": "Username",
+        "type": "string",
+        "order": 3
+      },
+      "password": {
+        "description": "The password associated with this username.",
+        "title": "Password",
+        "type": "string",
+        "airbyte_secret": true,
+        "order": 4
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\">JDBC URL parameters</a>.",
+        "title": "JDBC URL Parameters (Advanced)",
+        "type": "string",
+        "order": 5
+      },
+      "ssl": {
+        "title": "SSL Connection",
+        "description": "Encrypt data using SSL.",
+        "type": "boolean",
+        "default": true,
+        "order": 6
+      },
+      "tunnel_method": {
+        "type": "object",
+        "title": "SSH Tunnel Method",
+        "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.",
+        "oneOf": [
+          {
+            "title": "No Tunnel",
+            "required": ["tunnel_method"],
+            "properties": {
+              "tunnel_method": {
+                "description": "No ssh tunnel needed to connect to database",
+                "type": "string",
+                "const": "NO_TUNNEL",
+                "order": 0
+              }
+            }
+          },
+          {
+            "title": "SSH Key Authentication",
+            "required": [
+              "tunnel_method",
+              "tunnel_host",
+              "tunnel_port",
+              "tunnel_user",
+              "ssh_key"
+            ],
+            "properties": {
+              "tunnel_method": {
+                "description": "Connect through a jump server tunnel host using username and ssh key",
+                "type": "string",
+                "const": "SSH_KEY_AUTH",
+                "order": 0
+              },
+              "tunnel_host": {
+                "title": "SSH Tunnel Jump Server Host",
+                "description": "Hostname of the jump server host that allows inbound ssh tunnel.",
+                "type": "string",
+                "order": 1
+              },
+              "tunnel_port": {
+                "title": "SSH Connection Port",
+                "description": "Port on the proxy/jump server that accepts inbound ssh connections.",
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 65536,
+                "default": 22,
+                "examples": ["22"],
+                "order": 2
+              },
+              "tunnel_user": {
+                "title": "SSH Login Username",
+                "description": "OS-level username for logging into the jump server host.",
+                "type": "string",
+                "order": 3
+              },
+              "ssh_key": {
+                "title": "SSH Private Key",
+                "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )",
+                "type": "string",
+                "airbyte_secret": true,
+                "multiline": true,
+                "order": 4
+              }
+            }
+          },
+          {
+            "title": "Password Authentication",
+            "required": [
+              "tunnel_method",
+              "tunnel_host",
+              "tunnel_port",
+              "tunnel_user",
+              "tunnel_user_password"
+            ],
+            "properties": {
+              "tunnel_method": {
+                "description": "Connect through a jump server tunnel host using username and password authentication",
+                "type": "string",
+                "const": "SSH_PASSWORD_AUTH",
+                "order": 0
+              },
+              "tunnel_host": {
+                "title": "SSH Tunnel Jump Server Host",
+                "description": "Hostname of the jump server host that allows inbound ssh tunnel.",
+                "type": "string",
+                "order": 1
+              },
+              "tunnel_port": {
+                "title": "SSH Connection Port",
+                "description": "Port on the proxy/jump server that accepts inbound ssh connections.",
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 65536,
+                "default": 22,
+                "examples": ["22"],
+                "order": 2
+              },
+              "tunnel_user": {
+                "title": "SSH Login Username",
+                "description": "OS-level username for logging into the jump server host",
+                "type": "string",
+                "order": 3
+              },
+              "tunnel_user_password": {
+                "title": "Password",
+                "description": "OS-level password for logging into the jump server host",
+                "type": "string",
+                "airbyte_secret": true,
+                "order": 4
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "supportsNormalization": false,
+  "supportsDBT": false,
+  "supported_destination_sync_modes": []
+}

airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml

@@ -0,0 +1,7 @@
+# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-cockroachdb:dev
+tests:
+  spec:
+    - spec_path: "src/test-integration/resources/expected_spec.json"
+      config_path: "src/test-integration/resources/dummy_config.json"

airbyte-integrations/connectors/source-cockroachdb/acceptance-test-docker.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v /tmp:/tmp \
+  -v $(pwd):/test_input \
+  airbyte/source-acceptance-test \
+  --acceptance-test-config /test_input

airbyte-integrations/connectors/source-cockroachdb/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id 'application'
     id 'airbyte-docker'
     id 'airbyte-integration-test-java'
+    id 'airbyte-source-acceptance-test'
 }
 
 application {

airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """This fixture is a placeholder for external resources that acceptance test might require."""
+    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
+    yield
+    # TODO: clean up test dependencies

airbyte-integrations/connectors/source-cockroachdb/src/test-integration/resources/dummy_config.json

@@ -0,0 +1,6 @@
+{
+  "host": "default",
+  "port": 8123,
+  "database": "default",
+  "username": "default"
+}