Issue 3662: HTTP request source support 'csv type' response

airbyte-integrations/connectors/source-http-request/source_http_request/source.py

@@ -62,25 +62,42 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog:
         }
 
         # json body will be returned as the "data" stream". we can't know its schema ahead of time, so we assume it's object (i.e. valid json).
-        return AirbyteCatalog(streams=[AirbyteStream(name=SourceHttpRequest.STREAM_NAME, json_schema=json_schema)])
+        return AirbyteCatalog(
+            streams=[
+                AirbyteStream(
+                    name=SourceHttpRequest.STREAM_NAME, json_schema=json_schema
+                )
+            ]
+        )
 
     def read(
-        self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any]
+        self,
+        logger: AirbyteLogger,
+        config: json,
+        catalog: ConfiguredAirbyteCatalog,
+        state: Dict[str, any],
     ) -> Generator[AirbyteMessage, None, None]:
         r = self._make_request(config)
         if r.status_code != 200:
             raise Exception(f"Request failed. {r.text}")
 
         # need to eagerly fetch the json.
         data = r.json()
+
+        if isinstance(data, list) and isinstance(data[0], list):
+            # found list of list
+            data = [{"data": records} for records in data]
+
         if not isinstance(data, list):
             data = [data]
 
         for record in data:
             yield AirbyteMessage(
                 type=Type.RECORD,
                 record=AirbyteRecordMessage(
-                    stream=SourceHttpRequest.STREAM_NAME, data=record, emitted_at=int(datetime.now().timestamp()) * 1000
+                    stream=SourceHttpRequest.STREAM_NAME,
+                    data=record,
+                    emitted_at=int(datetime.now().timestamp()) * 1000,
                 ),
             )
 

airbyte-integrations/connectors/source-http-request/unit_tests/unit_test.py

@@ -25,9 +25,20 @@
 
 import unittest
 
+from unittest.mock import patch
+
 from source_http_request import SourceHttpRequest
 
 
+class MockResponse:
+    def __init__(self, json_data, status_code):
+        self.json_data = json_data
+        self.status_code = status_code
+
+    def json(self):
+        return self.json_data
+
+
 class TestSourceHttpRequest(unittest.TestCase):
     def test_parse_config(self):
         config = {
@@ -46,3 +57,35 @@ def test_parse_config(self):
             "body": {"something": "good"},
         }
         self.assertEqual(expected, actual)
+
+    def test_json_array_response(self):
+        with patch.object(
+            SourceHttpRequest,
+            attribute="_make_request",
+            return_value=MockResponse(
+                json_data=[
+                    ["foo", "bar"],
+                    ["test", 10],
+                    ["test2", 15],
+                ],
+                status_code=200,
+            ),
+        ):
+            expected = [
+                {"data": ["foo", "bar"]},
+                {"data": ["test", "10"]},
+                {"data": ["test2", "15"]},
+            ]
+            source = SourceHttpRequest()
+            results = [
+                r.record.data
+                for r in list(
+                    source.read(
+                        logger=None,
+                        state=None,
+                        catalog=None,
+                        config={},
+                    )
+                )
+            ]
+            self.assertEqual(expected, results)