Skip to content

Commit bb32c82

Browse files
VitaliiMaltsevvmaltsev
VitaliiMaltsev
and
vmaltsev
authored
🎉 Oracle Source: Add encryption options (#6616)
* OracleSource add encryption * add tests * add tests * remove debug option from Dockerfile * add docs and fix unit tests * remove System.out.println * fix checkstyle | remove 'encryption' from required fields * update timeout to 30 seconds * bump version to 0.3.6 * bump version to 0.3.7 Co-authored-by: vmaltsev <vitalii.maltsev@globallogic.com>
1 parent f13313e commit bb32c82

File tree

9 files changed

+262
-13
lines changed

9 files changed

+262
-13
lines changed

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22
"sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83",
33
"name": "Oracle DB",
44
"dockerRepository": "airbyte/source-oracle",
5-
"dockerImageTag": "0.3.5",
5+
"dockerImageTag": "0.3.7",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle"
77
}

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@
326326
- sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
327327
name: Oracle DB
328328
dockerRepository: airbyte/source-oracle
329-
dockerImageTag: 0.3.5
329+
dockerImageTag: 0.3.7
330330
documentationUrl: https://docs.airbyte.io/integrations/sources/oracle
331331
sourceType: database
332332
- sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071

‎airbyte-integrations/connectors/source-oracle/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
99

1010
RUN tar xf ${APPLICATION}.tar --strip-components=1
1111

12-
LABEL io.airbyte.version=0.3.5
12+
LABEL io.airbyte.version=0.3.7
1313
LABEL io.airbyte.name=airbyte/source-oracle

‎airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java

+73-4
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,15 @@
1515
import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
1616
import io.airbyte.integrations.source.relationaldb.TableInfo;
1717
import io.airbyte.protocol.models.CommonField;
18+
import java.io.IOException;
19+
import java.io.PrintWriter;
1820
import java.sql.JDBCType;
1921
import java.util.ArrayList;
2022
import java.util.List;
2123
import java.util.Locale;
2224
import java.util.Set;
25+
import java.util.concurrent.TimeUnit;
26+
import org.apache.commons.lang3.RandomStringUtils;
2327
import org.slf4j.Logger;
2428
import org.slf4j.LoggerFactory;
2529

@@ -31,18 +35,35 @@ public class OracleSource extends AbstractJdbcSource implements Source {
3135

3236
private List<String> schemas;
3337

38+
private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks";
39+
private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8);
40+
41+
enum Protocol {
42+
TCP,
43+
TCPS
44+
}
45+
3446
public OracleSource() {
3547
super(DRIVER_CLASS, new OracleJdbcStreamingQueryConfiguration());
3648
}
3749

3850
@Override
3951
public JsonNode toDatabaseConfig(JsonNode config) {
52+
List<String> additionalParameters = new ArrayList<>();
53+
54+
Protocol protocol = config.has("encryption")
55+
? obtainConnectionProtocol(config.get("encryption"), additionalParameters)
56+
: Protocol.TCP;
57+
final String connectionString = String.format(
58+
"jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=%s)(HOST=%s)(PORT=%s))(CONNECT_DATA=(SID=%s)))",
59+
protocol,
60+
config.get("host").asText(),
61+
config.get("port").asText(),
62+
config.get("sid").asText());
63+
4064
final ImmutableMap.Builder<Object, Object> configBuilder = ImmutableMap.builder()
4165
.put("username", config.get("username").asText())
42-
.put("jdbc_url", String.format("jdbc:oracle:thin:@//%s:%s/%s",
43-
config.get("host").asText(),
44-
config.get("port").asText(),
45-
config.get("sid").asText()));
66+
.put("jdbc_url", connectionString);
4667

4768
if (config.has("password")) {
4869
configBuilder.put("password", config.get("password").asText());
@@ -56,10 +77,58 @@ public JsonNode toDatabaseConfig(JsonNode config) {
5677
schemas.add(schema.asText());
5778
}
5879
}
80+
if (!additionalParameters.isEmpty()) {
81+
String connectionParams = String.join(";", additionalParameters);
82+
configBuilder.put("connection_properties", connectionParams);
83+
}
5984

6085
return Jsons.jsonNode(configBuilder.build());
6186
}
6287

88+
private Protocol obtainConnectionProtocol(JsonNode encryption, List<String> additionalParameters) {
89+
String encryptionMethod = encryption.get("encryption_method").asText();
90+
switch (encryptionMethod) {
91+
case "unencrypted" -> {
92+
return Protocol.TCP;
93+
}
94+
case "client_nne" -> {
95+
String algorithm = encryption.get("encryption_algorithm").asText();
96+
additionalParameters.add("oracle.net.encryption_client=REQUIRED");
97+
additionalParameters.add("oracle.net.encryption_types_client=( " + algorithm + " )");
98+
return Protocol.TCP;
99+
}
100+
case "encrypted_verify_certificate" -> {
101+
try {
102+
convertAndImportCertificate(encryption.get("ssl_certificate").asText());
103+
} catch (IOException | InterruptedException e) {
104+
throw new RuntimeException("Failed to import certificate into Java Keystore");
105+
}
106+
additionalParameters.add("javax.net.ssl.trustStore=" + KEY_STORE_FILE_PATH);
107+
additionalParameters.add("javax.net.ssl.trustStoreType=JKS");
108+
additionalParameters.add("javax.net.ssl.trustStorePassword=" + KEY_STORE_PASS);
109+
return Protocol.TCPS;
110+
}
111+
}
112+
throw new RuntimeException("Failed to obtain connection protocol from config" + encryption.asText());
113+
}
114+
115+
private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException {
116+
Runtime run = Runtime.getRuntime();
117+
try (PrintWriter out = new PrintWriter("certificate.pem")) {
118+
out.print(certificate);
119+
}
120+
runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);
121+
runProcess("keytool -import -alias rds-root -keystore " + KEY_STORE_FILE_PATH + " -file certificate.der -storepass " + KEY_STORE_PASS + " -noprompt", run);
122+
}
123+
124+
private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException {
125+
Process pr = run.exec(cmd);
126+
if (!pr.waitFor(30, TimeUnit.SECONDS)) {
127+
pr.destroy();
128+
throw new RuntimeException("Timeout while executing: " + cmd);
129+
} ;
130+
}
131+
63132
@Override
64133
public List<TableInfo<CommonField<JDBCType>>> discoverInternal(JdbcDatabase database) throws Exception {
65134
List<TableInfo<CommonField<JDBCType>>> internals = new ArrayList<>();

‎airbyte-integrations/connectors/source-oracle/src/main/resources/spec.json

+67-3
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,11 @@
1414
},
1515
"port": {
1616
"title": "Port",
17-
"description": "Port of the database.",
17+
"description": "Port of the database.\nOracle Corporations recommends the following port numbers:\n1521 - Default listening port for client connections to the listener. \n2484 - Recommended and officially registered listening port for client connections to the listener using TCP/IP with SSL",
1818
"type": "integer",
1919
"minimum": 0,
2020
"maximum": 65536,
21-
"default": 1521,
22-
"examples": ["1521"]
21+
"default": 1521
2322
},
2423
"sid": {
2524
"title": "SID (Oracle System Identifier)",
@@ -45,6 +44,71 @@
4544
},
4645
"minItems": 1,
4746
"uniqueItems": true
47+
},
48+
"encryption": {
49+
"title": "Encryption",
50+
"type": "object",
51+
"description": "Encryption method to use when communicating with the database",
52+
"order": 6,
53+
"oneOf": [
54+
{
55+
"title": "Unencrypted",
56+
"additionalProperties": false,
57+
"description": "Data transfer will not be encrypted.",
58+
"required": ["encryption_method"],
59+
"properties": {
60+
"encryption_method": {
61+
"type": "string",
62+
"const": "unencrypted",
63+
"enum": ["unencrypted"],
64+
"default": "unencrypted"
65+
}
66+
}
67+
},
68+
{
69+
"title": "Native Network Ecryption (NNE)",
70+
"additionalProperties": false,
71+
"description": "Native network encryption gives you the ability to encrypt database connections, without the configuration overhead of TCP/IP and SSL/TLS and without the need to open and listen on different ports.",
72+
"required": ["encryption_method"],
73+
"properties": {
74+
"encryption_method": {
75+
"type": "string",
76+
"const": "client_nne",
77+
"enum": ["client_nne"],
78+
"default": "client_nne"
79+
},
80+
"encryption_algorithm": {
81+
"type": "string",
82+
"description": "This parameter defines the encryption algorithm to be used",
83+
"title": "Encryption Algorithm",
84+
"default": "AES256",
85+
"enum": ["AES256", "RC4_56", "3DES168"]
86+
}
87+
}
88+
},
89+
{
90+
"title": "TLS Encrypted (verify certificate)",
91+
"additionalProperties": false,
92+
"description": "Verify and use the cert provided by the server.",
93+
"required": ["encryption_method", "ssl_certificate"],
94+
"properties": {
95+
"encryption_method": {
96+
"type": "string",
97+
"const": "encrypted_verify_certificate",
98+
"enum": ["encrypted_verify_certificate"],
99+
"default": "encrypted_verify_certificate"
100+
},
101+
"ssl_certificate": {
102+
"title": "SSL PEM file",
103+
"description": "Privacy Enhanced Mail (PEM) files are concatenated certificate containers frequently used in certificate installations",
104+
"type": "string",
105+
"airbyte_secret": true,
106+
"multiline": true,
107+
"order": 4
108+
}
109+
}
110+
}
111+
]
48112
}
49113
}
50114
}

‎airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,10 @@ public ImmutableMap.Builder<Object, Object> getBasicOracleDbConfigBuider(OracleC
103103
.put("password", db.getPassword())
104104
.put("port", db.getExposedPorts().get(0))
105105
.put("sid", db.getSid())
106-
.put("schemas", List.of("JDBC_SPACE"));
106+
.put("schemas", List.of("JDBC_SPACE"))
107+
.put("encryption", Jsons.jsonNode(ImmutableMap.builder()
108+
.put("encryption_method", "unencrypted")
109+
.build()));
107110
}
108111

109112
@Override

‎airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ public class OracleSourceAcceptanceTest extends SourceAcceptanceTest {
3030
private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME";
3131
private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS";
3232

33-
private OracleContainer container;
34-
private JsonNode config;
33+
protected OracleContainer container;
34+
protected JsonNode config;
3535

3636
@Override
3737
protected void setupEnvironment(TestDestinationEnv environment) throws Exception {
@@ -45,6 +45,9 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception
4545
.put("username", container.getUsername())
4646
.put("password", container.getPassword())
4747
.put("schemas", List.of("JDBC_SPACE"))
48+
.put("encryption", Jsons.jsonNode(ImmutableMap.builder()
49+
.put("encryption_method", "unencrypted")
50+
.build()))
4851
.build());
4952

5053
JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(),

‎airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
*/
4+
5+
package io.airbyte.integrations.source.oracle;
6+
7+
import static org.junit.jupiter.api.Assertions.assertEquals;
8+
import static org.junit.jupiter.api.Assertions.assertTrue;
9+
10+
import com.fasterxml.jackson.databind.JsonNode;
11+
import com.fasterxml.jackson.databind.node.ObjectNode;
12+
import com.google.common.collect.ImmutableMap;
13+
import io.airbyte.commons.json.Jsons;
14+
import io.airbyte.db.Databases;
15+
import io.airbyte.db.jdbc.JdbcDatabase;
16+
import java.sql.SQLException;
17+
import java.util.List;
18+
import java.util.stream.Collectors;
19+
import org.junit.jupiter.api.Test;
20+
21+
public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest {
22+
23+
@Test
24+
public void testEncrytion() throws SQLException {
25+
final JsonNode clone = Jsons.clone(getConfig());
26+
((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder()
27+
.put("encryption_method", "client_nne")
28+
.put("encryption_algorithm", "3DES168")
29+
.build()));
30+
31+
String algorithm = clone.get("encryption")
32+
.get("encryption_algorithm").asText();
33+
34+
JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(),
35+
clone.get("password").asText(),
36+
String.format("jdbc:oracle:thin:@//%s:%s/%s",
37+
clone.get("host").asText(),
38+
clone.get("port").asText(),
39+
clone.get("sid").asText()),
40+
"oracle.jdbc.driver.OracleDriver",
41+
"oracle.net.encryption_client=REQUIRED;" +
42+
"oracle.net.encryption_types_client=( "
43+
+ algorithm + " )");
44+
45+
String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)";
46+
List<JsonNode> collect = database.query(network_service_banner).collect(Collectors.toList());
47+
48+
assertTrue(collect.get(2).get("NETWORK_SERVICE_BANNER").asText()
49+
.contains("Oracle Advanced Security: " + algorithm + " encryption"));
50+
}
51+
52+
@Test
53+
public void testNoneEncrytion() throws SQLException {
54+
55+
JdbcDatabase database = Databases.createJdbcDatabase(config.get("username").asText(),
56+
config.get("password").asText(),
57+
String.format("jdbc:oracle:thin:@//%s:%s/%s",
58+
config.get("host").asText(),
59+
config.get("port").asText(),
60+
config.get("sid").asText()),
61+
"oracle.jdbc.driver.OracleDriver");
62+
63+
String network_service_banner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)";
64+
List<JsonNode> collect = database.query(network_service_banner).collect(Collectors.toList());
65+
66+
assertTrue(collect.get(1).get("NETWORK_SERVICE_BANNER").asText()
67+
.contains("Oracle Advanced Security: encryption"));
68+
}
69+
70+
@Test
71+
public void testCheckProtocol() throws SQLException {
72+
final JsonNode clone = Jsons.clone(getConfig());
73+
((ObjectNode) clone).put("encryption", Jsons.jsonNode(ImmutableMap.builder()
74+
.put("encryption_method", "client_nne")
75+
.put("encryption_algorithm", "AES256")
76+
.build()));
77+
78+
String algorithm = clone.get("encryption")
79+
.get("encryption_algorithm").asText();
80+
81+
JdbcDatabase database = Databases.createJdbcDatabase(clone.get("username").asText(),
82+
clone.get("password").asText(),
83+
String.format("jdbc:oracle:thin:@//%s:%s/%s",
84+
clone.get("host").asText(),
85+
clone.get("port").asText(),
86+
clone.get("sid").asText()),
87+
"oracle.jdbc.driver.OracleDriver",
88+
"oracle.net.encryption_client=REQUIRED;" +
89+
"oracle.net.encryption_types_client=( "
90+
+ algorithm + " )");
91+
92+
String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual";
93+
List<JsonNode> collect = database.query(network_service_banner).collect(Collectors.toList());
94+
95+
assertEquals("tcp", collect.get(0).get("NETWORK_PROTOCOL").asText());
96+
}
97+
98+
}

‎docs/integrations/sources/oracle.md

+12
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,22 @@ If you can't find the data type you are looking for or have any problems feel fr
123123

124124
If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings.
125125

126+
## Encryption Options
127+
128+
Airbite has the ability to connect to the Oracle source with 3 network connectivity options:
129+
130+
1.`Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form.
131+
2.`Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports.
132+
In this case, the *SQLNET.ENCRYPTION_CLIENT* option will always be set as *REQUIRED* by default: The client or server will only accept encrypted traffic,
133+
but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs.
134+
3.`TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification.
135+
To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field
136+
126137
## Changelog
127138

128139
| Version | Date | Pull Request | Subject |
129140
| :------ | :-------- | :----- | :------ |
141+
| 0.3.7 | 2021-10-01 | [6616](https://github.com/airbytehq/airbyte/pull/6616) | Added network encryption options |
130142
| 0.3.6 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps |
131143
| 0.3.5 | 2021-09-22 | [6356](https://github.com/airbytehq/airbyte/pull/6356) | Added option to connect to DB via SSH. |
132144
| 0.3.4 | 2021-09-01 | [6038](https://github.com/airbytehq/airbyte/pull/6038) | Remove automatic filtering of system schemas. |

0 commit comments

Comments
 (0)