From cc0fae9bc6c4bad5c852349a6be76b55fee221a6 Mon Sep 17 00:00:00 2001 From: Ryan Conaway Date: Mon, 3 Apr 2023 09:54:43 -0400 Subject: [PATCH 1/4] wip --- .../discovery_api/services/presto_service.ex | 22 ++++++++++++++++--- .../controllers/data_controller.ex | 4 ++-- .../services/presto_service_test.exs | 17 ++++++++++++++ .../data_controller_preview_test.exs | 22 +++++++++++++++++++ 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/apps/discovery_api/lib/discovery_api/services/presto_service.ex b/apps/discovery_api/lib/discovery_api/services/presto_service.ex index b30cef18d..0b55ccd0b 100644 --- a/apps/discovery_api/lib/discovery_api/services/presto_service.ex +++ b/apps/discovery_api/lib/discovery_api/services/presto_service.ex @@ -17,6 +17,7 @@ defmodule DiscoveryApi.Services.PrestoService do |> Prestige.query!("show columns from #{dataset_system_name}") |> Map.get(:rows) |> Enum.map(fn [column_name | _tail] -> column_name end) + |> remove_metadata_columns() end def is_select_statement?(statement) do @@ -131,10 +132,11 @@ defmodule DiscoveryApi.Services.PrestoService do [] -> {:error, "Table #{system_name} not found"} names -> {:ok, names} end + |> remove_metadata_columns() end - def build_query(params, system_name) do - column_string = Map.get(params, "columns", "*") + def build_query(params, system_name, columns) do + column_string = Map.get(params, "columns", Enum.join(columns, ", ")) |> IO.inspect(label: "Column String") ["SELECT"] |> build_columns(column_string) @@ -170,7 +172,7 @@ defmodule DiscoveryApi.Services.PrestoService do defp build_clause("groupBy", value), do: "GROUP BY #{value}" defp build_columns(clauses, column_string) do - cleaned_columns = column_string |> clean_columns() |> Enum.join(", ") + cleaned_columns = column_string |> clean_columns() |> Enum.join(", ") |> IO.inspect(label: "Before") clauses ++ [cleaned_columns] end @@ -179,4 +181,18 @@ defmodule DiscoveryApi.Services.PrestoService do |> String.split(",", trim: true) |> Enum.map(&String.trim/1) end + + defp remove_metadata_columns({:error, _reason} = error) do + error + end + + defp remove_metadata_columns({:ok, columns}) do + {:ok, remove_metadata_columns(columns)} + end + + defp remove_metadata_columns(columns) do + metadata_columns = ["_extraction_start_time", "_ingestion_id", "os_partition"] + + columns |> Enum.reject(fn column -> column in metadata_columns end) + end end diff --git a/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex b/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex index 934808ae6..3c77497d0 100644 --- a/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex +++ b/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex @@ -68,7 +68,7 @@ defmodule DiscoveryApiWeb.DataController do api_key = Plug.Conn.get_req_header(conn, "api_key") with {:ok, columns} <- PrestoService.get_column_names(session, dataset_name, Map.get(params, "columns")), - {:ok, query} <- PrestoService.build_query(params, dataset_name), + {:ok, query} <- PrestoService.build_query(params, dataset_name, columns), {:ok, affected_models} <- QueryAccessUtils.get_affected_models(query), true <- QueryAccessUtils.user_is_authorized?(affected_models, current_user, api_key) do data_stream = @@ -77,7 +77,7 @@ defmodule DiscoveryApiWeb.DataController do |> Stream.flat_map(&Prestige.Result.as_maps/1) rendered_data_stream = - DataView.render_as_stream(:data, format, %{stream: data_stream, columns: columns, dataset_name: dataset_name, schema: schema}) + DataView.render_as_stream(:data, format, %{stream: data_stream, columns: columns, dataset_name: dataset_name, schema: schema}) |> IO.inspect(label: "Results") resp_as_stream(conn, rendered_data_stream, format, dataset_id) else diff --git a/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs b/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs index fcf4e20f6..73f63cef4 100644 --- a/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs +++ b/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs @@ -43,6 +43,23 @@ defmodule DiscoveryApi.Services.PrestoServiceTest do assert list_of_columns == result end + test "preview_columns should filter out any metadata columns" do + dataset = "things_in_the_fire" + + list_of_columns = ["col_a"] + + unprocessed_columns = %Prestige.Result{ + columns: :doesnt_matter, + presto_headers: :doesnt_matter, + rows: [["col_a", "varchar", "", ""], ["_extraction_start_time", "varchar", "", ""], ["_ingestion_id", "varchar", "", ""], ["os_partition", "varchar", "", ""]] + } + + allow(Prestige.query!(:connection, "show columns from #{dataset}"), return: unprocessed_columns) + + result = PrestoService.preview_columns(:connection, dataset) + assert list_of_columns == result + end + describe "get_affected_tables/1" do setup do public_one_model = diff --git a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs index 30d95ba5f..8e74a8255 100644 --- a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs +++ b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs @@ -50,6 +50,28 @@ defmodule DiscoveryApiWeb.DataController.PreviewTest do assert expected == actual end + test "preview controller does not return any metadata columns", %{conn: conn, model: model} do + list_of_maps = [ + %{"id" => Faker.UUID.v4(), "_ingestion_id" => "will", "_extraction_id" => "be", "os_partition" => "removed", "json_encoded" => "{\"json_encoded\": \"tony\"}", "other" => "foo"}, + %{"id" => Faker.UUID.v4(), "_ingestion_id" => "will", "_extraction_id" => "be", "os_partition" => "removed", "json_encoded" => "{\"json_encoded\": \"andy\"}"}, + %{"id" => Faker.UUID.v4(), "_ingestion_id" => "will", "_extraction_id" => "be", "os_partition" => "removed", "json_encoded" => "{\"json_encoded\": \"smith\"}"} + ] + + schema = model.schema + encoded_maps = Enum.map(list_of_maps, &JsonFieldDecoder.decode_one_datum(schema, &1)) + + list_of_columns = ["id", "json_encoded", "other"] + + expected = %{"data" => encoded_maps, "meta" => %{"columns" => list_of_columns}} + + expect(PrestoService.preview(any(), @system_name), return: list_of_maps) + expect(PrestoService.preview_columns(any(), @system_name), return: list_of_columns) + + actual = conn |> put_req_header("accept", "application/json") |> get("/api/v1/dataset/#{@dataset_id}/preview") |> json_response(200) + + assert expected == actual + end + test "preview controller returns an empty list for an existing dataset with no data", %{conn: conn} do list_of_columns = ["id", "json_encoded"] expected = %{"data" => [], "meta" => %{"columns" => list_of_columns}} From e08596942e5a93e5896dae6cc582c67f8bd284ab Mon Sep 17 00:00:00 2001 From: Ryan Conaway Date: Mon, 3 Apr 2023 10:24:28 -0400 Subject: [PATCH 2/4] Building sql query based on columns rather than SELECT * --- .../controllers/data_controller.ex | 2 +- .../data_controller_content_test.exs | 2 +- .../data_controller_query_test.exs | 30 +++++++++---------- .../data_controller_restricted_test.exs | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex b/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex index 3c77497d0..b6ff1e69f 100644 --- a/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex +++ b/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex @@ -68,7 +68,7 @@ defmodule DiscoveryApiWeb.DataController do api_key = Plug.Conn.get_req_header(conn, "api_key") with {:ok, columns} <- PrestoService.get_column_names(session, dataset_name, Map.get(params, "columns")), - {:ok, query} <- PrestoService.build_query(params, dataset_name, columns), + {:ok, query} <- PrestoService.build_query(params, dataset_name, columns) |> IO.inspect(label: "RYAN - BUILT QUERY"), {:ok, affected_models} <- QueryAccessUtils.get_affected_models(query), true <- QueryAccessUtils.user_is_authorized?(affected_models, current_user, api_key) do data_stream = diff --git a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_content_test.exs b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_content_test.exs index b21039ecd..3a62f3f3b 100644 --- a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_content_test.exs +++ b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_content_test.exs @@ -68,7 +68,7 @@ defmodule DiscoveryApiWeb.DataController.ContentTest do allow(PrestoService.get_column_names(any(), any(), any()), return: {:ok, ["feature"]}) allow(PrestoService.preview_columns(any(), @system_name), return: ["feature"]) allow(PrestoService.preview(any(), @system_name), return: @geo_json_features) - allow(PrestoService.build_query(any(), any()), return: {:ok, "select * from #{@system_name}"}) + allow(PrestoService.build_query(any(), any(), any()), return: {:ok, "select * from #{@system_name}"}) allow(Prestige.new_session(any()), return: :connect) allow(Prestige.query!(any(), "select * from #{@system_name}"), return: :result) diff --git a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs index b54e7e30a..dc9f4d350 100644 --- a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs +++ b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs @@ -62,7 +62,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do conn |> put_req_header("accept", "text/csv") |> get(url) |> response(200) assert_called(Prestige.query!(:connection, "describe #{@system_name}"), once()) - assert_called(Prestige.stream!(:connection, "SELECT * FROM #{@system_name}"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name}"), once()) where( url: [ @@ -75,7 +75,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do data_test "selects using the where clause provided", %{conn: conn} do conn |> put_req_header("accept", "text/csv") |> get(url, where: "name='Robby'") |> response(200) - assert_called(Prestige.stream!(:connection, "SELECT * FROM #{@system_name} WHERE name='Robby'"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} WHERE name='Robby'"), once()) where( url: [ @@ -88,7 +88,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do data_test "selects using the order by clause provided", %{conn: conn} do conn |> put_req_header("accept", "text/csv") |> get(url, orderBy: "id") |> response(200) - assert_called(Prestige.stream!(:connection, "SELECT * FROM #{@system_name} ORDER BY id"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} ORDER BY id"), once()) where( url: [ @@ -101,7 +101,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do data_test "selects using the limit clause provided", %{conn: conn} do conn |> put_req_header("accept", "text/csv") |> get(url, limit: "200") |> response(200) - assert_called(Prestige.stream!(:connection, "SELECT * FROM #{@system_name} LIMIT 200"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} LIMIT 200"), once()) where( url: [ @@ -114,7 +114,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do data_test "selects using the group by clause provided", %{conn: conn} do conn |> put_req_header("accept", "text/csv") |> get(url, groupBy: "one") |> response(200) - assert_called(Prestige.stream!(:connection, "SELECT * FROM #{@system_name} GROUP BY one"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} GROUP BY one"), once()) where( url: [ @@ -130,7 +130,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do |> get(url, where: "id=1", orderBy: "name", limit: "200", groupBy: "name") |> response(200) - assert_called(Prestige.stream!(:connection, "SELECT * FROM #{@system_name} WHERE id=1 GROUP BY name ORDER BY name LIMIT 200"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} WHERE id=1 GROUP BY name ORDER BY name LIMIT 200"), once()) where( url: [ @@ -198,11 +198,11 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do test "json queries cannot contain semicolons", %{conn: conn} do conn |> put_req_header("accept", "application/json") - |> get("/api/v1/organization/org1/dataset/data1/query", columns: "id,one; select * from system; two") + |> get("/api/v1/organization/org1/dataset/data1/query", columns: "id,one; select id, name from system; two") |> response(400) assert_called( - Prestige.stream!(:connection, "SELECT id, one; select * from system; two FROM coda__test_dataset"), + Prestige.stream!(:connection, "SELECT id, one; select id, name from system; two FROM coda__test_dataset"), times(0) ) end @@ -210,17 +210,17 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do test "csv queries cannot contain semicolons", %{conn: conn} do conn |> put_req_header("accept", "text/csv") - |> get("/api/v1/organization/org1/dataset/data1/query", columns: "id,one; select * from system; two") + |> get("/api/v1/organization/org1/dataset/data1/query", columns: "id,one; select id, name from system; two") |> response(400) assert_called( - Prestige.stream!(:connection, "SELECT id, one; select * from system; two FROM coda__test_dataset"), + Prestige.stream!(:connection, "SELECT id, one; select id, name from system; two FROM coda__test_dataset"), times(0) ) end test "queries cannot contain block comments", %{conn: conn} do - query_string = "SELECT * FROM coda__test_dataset ORDER BY /* This is a comment */" + query_string = "SELECT id, name FROM coda__test_dataset ORDER BY /* This is a comment */" conn |> put_req_header("accept", "text/csv") @@ -231,7 +231,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do end test "queries cannot contain single-line comments", %{conn: conn} do - query_string = "SELECT * FROM coda__test_dataset ORDER BY -- This is a comment" + query_string = "SELECT id, name FROM coda__test_dataset ORDER BY -- This is a comment" conn |> put_req_header("accept", "text/csv") @@ -260,14 +260,14 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do allow(Model.get(any()), return: model) allow(Model.get_all(), return: [model]) - allow(Prestige.stream!(:connection, "SELECT * FROM geojson"), + allow(Prestige.stream!(:connection, "SELECT id, name FROM geojson"), return: [:any] ) allow(Redix.command!(any(), any()), return: :doesnt_matter) allow(PrestoService.get_column_names(any(), any(), any()), return: {:ok, ["feature"]}) - allow(PrestoService.build_query(any(), any()), return: {:ok, "SELECT * FROM geojson"}) + allow(PrestoService.build_query(any(), any(), any()), return: {:ok, "SELECT id, name FROM geojson"}) allow(PrestoService.is_select_statement?(any()), return: true) allow(PrestoService.get_affected_tables(any(), any()), return: {:ok, ["geojson__geojson"]}) allow(ModelAccessUtils.has_access?(any(), any()), return: true) @@ -307,7 +307,7 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do "type" => @feature_type } - assert_called(Prestige.stream!(:connection, "SELECT * FROM geojson"), once()) + assert_called(Prestige.stream!(:connection, "SELECT id, name FROM geojson"), once()) where( url: [ diff --git a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_restricted_test.exs b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_restricted_test.exs index 67964fb1b..a213b3f2d 100644 --- a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_restricted_test.exs +++ b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_restricted_test.exs @@ -45,7 +45,7 @@ defmodule DiscoveryApiWeb.DataController.RestrictedTest do allow(PrestoService.get_column_names(any(), any(), any()), return: {:ok, ["id", "name"]}) allow(PrestoService.preview_columns(any(), @system_name), return: ["id", "name"]) allow(PrestoService.preview(any(), @system_name), return: [[1, "Joe"], [2, "Robby"]]) - allow(PrestoService.build_query(any(), any()), return: {:ok, "select * from #{@system_name}"}) + allow(PrestoService.build_query(any(), any(), any()), return: {:ok, "select * from #{@system_name}"}) allow(PrestoService.is_select_statement?("select * from #{@system_name}"), return: true) allow(PrestoService.get_affected_tables(any(), "select * from #{@system_name}"), return: {:ok, ["#{@system_name}"]}) From 20dfb09f35362fbe4cc1a7611988b6dd4902b51d Mon Sep 17 00:00:00 2001 From: Ryan Conaway Date: Mon, 3 Apr 2023 10:26:47 -0400 Subject: [PATCH 3/4] removed debug statements --- .../lib/discovery_api/services/presto_service.ex | 4 ++-- .../lib/discovery_api_web/controllers/data_controller.ex | 4 ++-- apps/raptor_service/lib/raptor_service.ex | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/discovery_api/lib/discovery_api/services/presto_service.ex b/apps/discovery_api/lib/discovery_api/services/presto_service.ex index 0b55ccd0b..a0bb8eb4f 100644 --- a/apps/discovery_api/lib/discovery_api/services/presto_service.ex +++ b/apps/discovery_api/lib/discovery_api/services/presto_service.ex @@ -136,7 +136,7 @@ defmodule DiscoveryApi.Services.PrestoService do end def build_query(params, system_name, columns) do - column_string = Map.get(params, "columns", Enum.join(columns, ", ")) |> IO.inspect(label: "Column String") + column_string = Map.get(params, "columns", Enum.join(columns, ", ")) ["SELECT"] |> build_columns(column_string) @@ -172,7 +172,7 @@ defmodule DiscoveryApi.Services.PrestoService do defp build_clause("groupBy", value), do: "GROUP BY #{value}" defp build_columns(clauses, column_string) do - cleaned_columns = column_string |> clean_columns() |> Enum.join(", ") |> IO.inspect(label: "Before") + cleaned_columns = column_string |> clean_columns() |> Enum.join(", ") clauses ++ [cleaned_columns] end diff --git a/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex b/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex index b6ff1e69f..37a8f6fc6 100644 --- a/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex +++ b/apps/discovery_api/lib/discovery_api_web/controllers/data_controller.ex @@ -68,7 +68,7 @@ defmodule DiscoveryApiWeb.DataController do api_key = Plug.Conn.get_req_header(conn, "api_key") with {:ok, columns} <- PrestoService.get_column_names(session, dataset_name, Map.get(params, "columns")), - {:ok, query} <- PrestoService.build_query(params, dataset_name, columns) |> IO.inspect(label: "RYAN - BUILT QUERY"), + {:ok, query} <- PrestoService.build_query(params, dataset_name, columns), {:ok, affected_models} <- QueryAccessUtils.get_affected_models(query), true <- QueryAccessUtils.user_is_authorized?(affected_models, current_user, api_key) do data_stream = @@ -77,7 +77,7 @@ defmodule DiscoveryApiWeb.DataController do |> Stream.flat_map(&Prestige.Result.as_maps/1) rendered_data_stream = - DataView.render_as_stream(:data, format, %{stream: data_stream, columns: columns, dataset_name: dataset_name, schema: schema}) |> IO.inspect(label: "Results") + DataView.render_as_stream(:data, format, %{stream: data_stream, columns: columns, dataset_name: dataset_name, schema: schema}) resp_as_stream(conn, rendered_data_stream, format, dataset_id) else diff --git a/apps/raptor_service/lib/raptor_service.ex b/apps/raptor_service/lib/raptor_service.ex index ac7f267c7..8c7fe1f89 100644 --- a/apps/raptor_service/lib/raptor_service.ex +++ b/apps/raptor_service/lib/raptor_service.ex @@ -102,7 +102,7 @@ defmodule RaptorService do end def check_auth0_role(raptor_url, api_key, role) do - case HTTPoison.get(url_for_checking_role(raptor_url, api_key, role) |> IO.inspect(label: "RYAN - URL")) do + case HTTPoison.get(url_for_checking_role(raptor_url, api_key, role)) do {:ok, %{body: body, status_code: status_code}} when status_code in 200..399 -> {:ok, Jason.decode!(body)["has_role"]} From c414ce251543d4e3ba13a58baee80953172e22d4 Mon Sep 17 00:00:00 2001 From: Ryan Conaway Date: Mon, 3 Apr 2023 10:29:26 -0400 Subject: [PATCH 4/4] formatting --- .../services/presto_service_test.exs | 7 +++++- .../data_controller_preview_test.exs | 25 ++++++++++++++++--- .../data_controller_query_test.exs | 5 +++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs b/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs index 73f63cef4..f549f8bbc 100644 --- a/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs +++ b/apps/discovery_api/test/unit/discovery_api/services/presto_service_test.exs @@ -51,7 +51,12 @@ defmodule DiscoveryApi.Services.PrestoServiceTest do unprocessed_columns = %Prestige.Result{ columns: :doesnt_matter, presto_headers: :doesnt_matter, - rows: [["col_a", "varchar", "", ""], ["_extraction_start_time", "varchar", "", ""], ["_ingestion_id", "varchar", "", ""], ["os_partition", "varchar", "", ""]] + rows: [ + ["col_a", "varchar", "", ""], + ["_extraction_start_time", "varchar", "", ""], + ["_ingestion_id", "varchar", "", ""], + ["os_partition", "varchar", "", ""] + ] } allow(Prestige.query!(:connection, "show columns from #{dataset}"), return: unprocessed_columns) diff --git a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs index 8e74a8255..10a4489e4 100644 --- a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs +++ b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_preview_test.exs @@ -52,9 +52,28 @@ defmodule DiscoveryApiWeb.DataController.PreviewTest do test "preview controller does not return any metadata columns", %{conn: conn, model: model} do list_of_maps = [ - %{"id" => Faker.UUID.v4(), "_ingestion_id" => "will", "_extraction_id" => "be", "os_partition" => "removed", "json_encoded" => "{\"json_encoded\": \"tony\"}", "other" => "foo"}, - %{"id" => Faker.UUID.v4(), "_ingestion_id" => "will", "_extraction_id" => "be", "os_partition" => "removed", "json_encoded" => "{\"json_encoded\": \"andy\"}"}, - %{"id" => Faker.UUID.v4(), "_ingestion_id" => "will", "_extraction_id" => "be", "os_partition" => "removed", "json_encoded" => "{\"json_encoded\": \"smith\"}"} + %{ + "id" => Faker.UUID.v4(), + "_ingestion_id" => "will", + "_extraction_id" => "be", + "os_partition" => "removed", + "json_encoded" => "{\"json_encoded\": \"tony\"}", + "other" => "foo" + }, + %{ + "id" => Faker.UUID.v4(), + "_ingestion_id" => "will", + "_extraction_id" => "be", + "os_partition" => "removed", + "json_encoded" => "{\"json_encoded\": \"andy\"}" + }, + %{ + "id" => Faker.UUID.v4(), + "_ingestion_id" => "will", + "_extraction_id" => "be", + "os_partition" => "removed", + "json_encoded" => "{\"json_encoded\": \"smith\"}" + } ] schema = model.schema diff --git a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs index dc9f4d350..40b7b61f4 100644 --- a/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs +++ b/apps/discovery_api/test/unit/discovery_api_web/controllers/data_controller_query_test.exs @@ -130,7 +130,10 @@ defmodule DiscoveryApiWeb.DataController.QueryTest do |> get(url, where: "id=1", orderBy: "name", limit: "200", groupBy: "name") |> response(200) - assert_called(Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} WHERE id=1 GROUP BY name ORDER BY name LIMIT 200"), once()) + assert_called( + Prestige.stream!(:connection, "SELECT id, name FROM #{@system_name} WHERE id=1 GROUP BY name ORDER BY name LIMIT 200"), + once() + ) where( url: [