From e21780d0be0f2d2048cb5195d03157b2ca75a39d Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Mon, 27 Mar 2023 16:45:15 -0500 Subject: [PATCH 01/11] enable XML body in HTTP post --- .../lib/andi/input_schemas/input_converter.ex | 43 +++++++------- .../input_schemas/input_converter_test.exs | 58 ++++--------------- .../lib/reaper/data_extract/extract_step.ex | 5 +- apps/reaper/lib/reaper/http/downloader.ex | 18 +++++- apps/reaper/lib/reaper/url_builder.ex | 16 +++++ .../reaper/data_extract/extract_step_test.exs | 32 +++++----- .../test/unit/reaper/http/downloader_test.exs | 46 +++++++++++++++ .../test/unit/reaper/url_builder_test.exs | 40 +++++++++++++ 8 files changed, 166 insertions(+), 92 deletions(-) diff --git a/apps/andi/lib/andi/input_schemas/input_converter.ex b/apps/andi/lib/andi/input_schemas/input_converter.ex index d17322b6c..40626fff2 100644 --- a/apps/andi/lib/andi/input_schemas/input_converter.ex +++ b/apps/andi/lib/andi/input_schemas/input_converter.ex @@ -254,42 +254,48 @@ defmodule Andi.InputSchemas.InputConverter do defp update_context_from_smrt_step(context, "http") do context - |> encode_extract_step_body_as_json() + |> validate_extract_step_body() |> Map.update(:queryParams, [], &to_key_value_list/1) |> Map.update(:headers, [], &to_key_value_list/1) end defp update_context_from_smrt_step(context, type) when type in ["s3", "auth"] do context - |> encode_extract_step_body_as_json() + |> validate_extract_step_body() |> Map.update(:headers, [], &to_key_value_list/1) end defp update_context_from_smrt_step(context, _), do: context - defp encode_extract_step_body_as_json(%{body: body} = smrt_extract_step) when body not in ["", nil] do + defp validate_extract_step_body(%{body: body} = smrt_extract_step) when body not in ["", nil] do case body do _ when is_binary(body) -> - raise_error_if_invalid_json(body) + raise_error_if_invalid_body(body) Map.put(smrt_extract_step, :body, body) - _ when is_map(body) -> - Map.put(smrt_extract_step, :body, Jason.encode!(body)) - - _ when is_list(body) -> - Map.put(smrt_extract_step, :body, Jason.encode!(body)) - _ -> - Logger.error("Received an extract step body that is not a string or a map. Received body: #{inspect(body)}") + Logger.error("Received an extract step body that is not able to be encoded as json or xml. Received body: #{inspect(body)}") smrt_extract_step end end - defp raise_error_if_invalid_json(payload) do - Jason.decode!(payload) + defp raise_error_if_invalid_body(body) do + case Jason.decode(body) do + {:ok, _} -> + body + + {:error, _} -> + try do + SweetXml.parse(body) + body + catch + :exit, _ -> + raise ArgumentError, message: "could not parse json or xml: #{inspect(body)}" + end + end end - defp encode_extract_step_body_as_json(smrt_extract_step), do: smrt_extract_step + defp validate_extract_step_body(smrt_extract_step), do: smrt_extract_step defp add_dataset_id(schema, dataset_id, parent_bread_crumb \\ "") do bread_crumb = parent_bread_crumb <> schema.name @@ -357,7 +363,6 @@ defmodule Andi.InputSchemas.InputConverter do defp update_context_from_andi_step(context, "http") do context |> cast_keys_to_atom_in_map_recursively() - |> decode_andi_extract_step_body() |> Map.put_new(:body, %{}) |> Map.put_new(:protocol, nil) |> Map.update(:queryParams, nil, &convert_key_value_to_map/1) @@ -403,7 +408,6 @@ defmodule Andi.InputSchemas.InputConverter do defp update_context_from_andi_step(context, "s3") do context - |> decode_andi_extract_step_body() |> Map.update(:headers, nil, &convert_key_value_to_map/1) end @@ -415,7 +419,6 @@ defmodule Andi.InputSchemas.InputConverter do defp update_context_from_andi_step(context, "auth") do context - |> decode_andi_extract_step_body() |> Map.put_new(:body, %{}) |> Map.put_new(:encodeMethod, "json") |> Map.update(:headers, nil, &convert_key_value_to_map/1) @@ -426,12 +429,6 @@ defmodule Andi.InputSchemas.InputConverter do defp ensure_nil_unit(""), do: nil defp ensure_nil_unit(unit), do: unit - defp decode_andi_extract_step_body(%{body: body} = http_extract_step) when body not in ["", nil] do - Map.put(http_extract_step, :body, Jason.decode!(body)) - end - - defp decode_andi_extract_step_body(andi_extract_step), do: andi_extract_step - defp populate_schema_field_default(field) do {use_default, updated_field} = Map.pop(field, :use_default) {offset, updated_field} = Map.pop(updated_field, :default_offset) diff --git a/apps/andi/test/unit/andi/input_schemas/input_converter_test.exs b/apps/andi/test/unit/andi/input_schemas/input_converter_test.exs index 08a316bd4..ec0a90dac 100644 --- a/apps/andi/test/unit/andi/input_schemas/input_converter_test.exs +++ b/apps/andi/test/unit/andi/input_schemas/input_converter_test.exs @@ -40,36 +40,7 @@ defmodule Andi.InputSchemas.InputConverterTest do ] end - test "prepare_smrt_ingestion_for_casting json encodes the extract step body if it is a map" do - smrt_ingestion = - TDG.create_ingestion(%{ - extractSteps: [ - %{ - type: "s3", - context: %{ - url: "123.com", - body: %{foo: 123}, - headers: %{"api-key" => "to-my-heart"} - } - } - ] - }) - - result = InputConverter.prepare_smrt_ingestion_for_casting(smrt_ingestion) - - assert result.extractSteps == [ - %{ - context: %{ - body: "{\"foo\":123}", - headers: [%{key: "api-key", value: "to-my-heart"}], - url: "123.com" - }, - type: "s3" - } - ] - end - - test "prepare_smrt_ingestion_for_casting json encodes the extract step body if it is a list" do + test "prepare_smrt_ingestion_for_casting throws an error if a body is not valid json" do smrt_ingestion = TDG.create_ingestion(%{ extractSteps: [ @@ -77,28 +48,23 @@ defmodule Andi.InputSchemas.InputConverterTest do type: "s3", context: %{ url: "123.com", - body: [%{foo: 123}], + body: "{foo:123}", headers: %{"api-key" => "to-my-heart"} } } ] }) - result = InputConverter.prepare_smrt_ingestion_for_casting(smrt_ingestion) - - assert result.extractSteps == [ - %{ - context: %{ - body: "[{\"foo\":123}]", - headers: [%{key: "api-key", value: "to-my-heart"}], - url: "123.com" - }, - type: "s3" - } - ] + try do + result = InputConverter.prepare_smrt_ingestion_for_casting(smrt_ingestion) + flunk("Expected incorrectly formatted body to raise an error") + rescue + e in ArgumentError -> + :ok + end end - test "prepare_smrt_ingestion_for_casting throws an error if a body is not valid json" do + test "prepare_smrt_ingestion_for_casting throws an error if a body is not valid xml" do smrt_ingestion = TDG.create_ingestion(%{ extractSteps: [ @@ -106,7 +72,7 @@ defmodule Andi.InputSchemas.InputConverterTest do type: "s3", context: %{ url: "123.com", - body: "{foo:123}", + body: "test", headers: %{"api-key" => "to-my-heart"} } } @@ -117,7 +83,7 @@ defmodule Andi.InputSchemas.InputConverterTest do result = InputConverter.prepare_smrt_ingestion_for_casting(smrt_ingestion) flunk("Expected incorrectly formatted body to raise an error") rescue - e in Jason.DecodeError -> + e in ArgumentError -> :ok end end diff --git a/apps/reaper/lib/reaper/data_extract/extract_step.ex b/apps/reaper/lib/reaper/data_extract/extract_step.ex index faf2699d1..7a19a1dd2 100644 --- a/apps/reaper/lib/reaper/data_extract/extract_step.ex +++ b/apps/reaper/lib/reaper/data_extract/extract_step.ex @@ -100,9 +100,6 @@ defmodule Reaper.DataExtract.ExtractStep do defp process_body(body, _assigns) when body in ["", nil], do: "" defp process_body(body, assigns) do - body - |> UrlBuilder.safe_evaluate_parameters(assigns) - |> Enum.into(%{}) - |> Jason.encode!() + body |> UrlBuilder.safe_evaluate_body(assigns) end end diff --git a/apps/reaper/lib/reaper/http/downloader.ex b/apps/reaper/lib/reaper/http/downloader.ex index b5851f3a2..ef15fd1d9 100644 --- a/apps/reaper/lib/reaper/http/downloader.ex +++ b/apps/reaper/lib/reaper/http/downloader.ex @@ -211,10 +211,22 @@ defmodule Reaper.Http.Downloader do {to_string(key), EEx.eval_string(value, [])} end - # Right now we assume any body sent in a request will be json encoded. We may change this in the future but - # at this time we dont have any use cases to do otherwise defp add_content_type(headers, ""), do: headers - defp add_content_type(headers, _body), do: [{"Content-Type", "application/json"} | headers] + defp add_content_type(headers, body) do + case Jason.decode(body) do + {:ok, _} -> + [{"Content-Type", "application/json"} | headers] + + {:error, _} -> + try do + SweetXml.parse(body) + [{"Content-Type", "text/xml"} | headers] + catch + :exit, _ -> + raise ArgumentError, message: "body is not in json or xml format: #{inspect(body)}" + end + end + end defp handle_compression(headers, file_name) do content_type = Util.get_header_value(headers, "content-encoding") || Util.get_header_value(headers, "content-type") diff --git a/apps/reaper/lib/reaper/url_builder.ex b/apps/reaper/lib/reaper/url_builder.ex index aded85582..a832c1ef1 100644 --- a/apps/reaper/lib/reaper/url_builder.ex +++ b/apps/reaper/lib/reaper/url_builder.ex @@ -29,6 +29,22 @@ defmodule Reaper.UrlBuilder do end) end + def safe_evaluate_body(body, bindings) when is_binary(body) do + regex = ~r"{{(.+?)}}" + replacements = Regex.scan(regex, body) + + value = + Enum.reduce(replacements, body, fn replacement, new_body -> + Regex.replace( + ~r"#{List.first(replacement)}", + new_body, + bindings[String.to_atom(List.last(replacement))] + ) + end) + + value + end + def safe_evaluate_parameters(parameters, bindings) do Enum.map( parameters, diff --git a/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs b/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs index 811003f26..ddbc282d2 100644 --- a/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs +++ b/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs @@ -76,7 +76,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}", encodeMethod: "json", - body: %{Key: "AuthToken"}, + body: "{\"Key\": \"AuthToken\"}", headers: %{}, cacheTtl: nil }, @@ -113,7 +113,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}", encodeMethod: "json", - body: %{Key: "AuthToken"}, + body: "{\"Key\": \"AuthToken\"}", headers: %{}, cacheTtl: nil }, @@ -150,7 +150,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}", encodeMethod: "json", - body: %{Key: "AuthToken"}, + body: "{\"Key\": \"AuthToken\"}", headers: %{}, cacheTtl: nil }, @@ -182,7 +182,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}", encodeMethod: "json", - body: %{Key: "{{key}}"}, + body: "{\"Key\": \"{{key}}\"}", headers: %{}, cacheTtl: nil }, @@ -247,7 +247,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}/headers", encodeMethod: "json", - body: %{}, + body: "", headers: %{Header: "{{header}}"}, cacheTtl: nil }, @@ -275,7 +275,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}/{{path}}", encodeMethod: "json", - body: %{}, + body: "", headers: %{}, cacheTtl: nil }, @@ -303,7 +303,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do destination: "token", url: "http://localhost:#{bypass.port}", encodeMethod: "json", - body: %{Key: "AuthToken"}, + body: "{\"Key\": \"AuthToken\"}", headers: %{}, cacheTtl: nil }, @@ -424,7 +424,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do context: %{ action: "GET", protocol: nil, - body: %{}, + body: "", url: "#{sourceUrl}", queryParams: %{}, headers: %{} @@ -460,7 +460,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do context: %{ action: "GET", protocol: nil, - body: %{}, + body: "", url: "#{sourceUrl}/query", queryParams: %{ token: "{{token}}" @@ -492,7 +492,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do context: %{ action: "GET", protocol: nil, - body: %{}, + body: "", url: "#{sourceUrl}/headers", queryParams: %{}, headers: %{Bearer: "{{token}}"} @@ -518,7 +518,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do context: %{ action: "GET", protocol: nil, - body: %{}, + body: "", url: "#{sourceUrl}/{{path}}", queryParams: %{}, headers: %{} @@ -550,11 +550,11 @@ defmodule Reaper.DataExtract.ExtractStepTest do context: %{ action: "POST", protocol: nil, - body: %{ - soap_request: %{ - date: "{{date}}" + body: "{ + \"soap_request\": { + \"date\": \"{{date}}\" } - }, + }", url: "#{sourceUrl}/post", queryParams: %{}, headers: %{} @@ -582,7 +582,7 @@ defmodule Reaper.DataExtract.ExtractStepTest do context: %{ action: "GET", protocol: ["http1"], - body: %{}, + body: "", url: "#{sourceUrl}", queryParams: %{}, headers: %{} diff --git a/apps/reaper/test/unit/reaper/http/downloader_test.exs b/apps/reaper/test/unit/reaper/http/downloader_test.exs index c46594a5c..bfaeae24f 100644 --- a/apps/reaper/test/unit/reaper/http/downloader_test.exs +++ b/apps/reaper/test/unit/reaper/http/downloader_test.exs @@ -254,6 +254,52 @@ defmodule Reaper.Http.DownloaderTest do assert_called(Mint.HTTP.request(any(), any(), any(), evaluated_headers, any()), once()) end + test "adds json header when body is in json format", %{bypass: bypass} do + allow(Mint.HTTP.request(:connection, any(), any(), any(), any()), return: :ok) + + path = "/some.url" + url = "http://localhost:#{bypass.port}#{path}" + body = "{\"testKey\": \"testValue\"}" + + Bypass.stub(bypass, "GET", path, fn conn -> + Plug.Conn.resp(conn, 200, "data") + end) + + headers = %{ + "testKey" => "<%= Date.to_iso8601(~D[1970-01-02], :basic) %>", + :testB => "valB" + } + + evaluated_headers = [{"Content-Type", "application/json"}, {"testB", "valB"}, {"testKey", "19700102"}] + + {:ok, _} = Downloader.download(url, headers, to: "test.output", body: body) + + assert_called(Mint.HTTP.request(any(), any(), any(), evaluated_headers, any()), once()) + end + + test "adds xml header when body is in json format", %{bypass: bypass} do + allow(Mint.HTTP.request(:connection, any(), any(), any(), any()), return: :ok) + + path = "/some.url" + url = "http://localhost:#{bypass.port}#{path}" + body = "testValue" + + Bypass.stub(bypass, "GET", path, fn conn -> + Plug.Conn.resp(conn, 200, "data") + end) + + headers = %{ + "testKey" => "<%= Date.to_iso8601(~D[1970-01-02], :basic) %>", + :testB => "valB" + } + + evaluated_headers = [{"Content-Type", "text/xml"}, {"testB", "valB"}, {"testKey", "19700102"}] + + {:ok, _} = Downloader.download(url, headers, to: "test.output", body: body) + + assert_called(Mint.HTTP.request(any(), any(), any(), evaluated_headers, any()), once()) + end + test "protocol is used for connection", %{bypass: bypass} do allow(Mint.HTTP.connect(:connection, any(), any(), any()), return: :ok, meck_options: [:passthrough]) path = "/some.url" diff --git a/apps/reaper/test/unit/reaper/url_builder_test.exs b/apps/reaper/test/unit/reaper/url_builder_test.exs index 101b21932..f67543a33 100644 --- a/apps/reaper/test/unit/reaper/url_builder_test.exs +++ b/apps/reaper/test/unit/reaper/url_builder_test.exs @@ -72,4 +72,44 @@ defmodule Reaper.UrlBuilderTest do ] ]) end + + test "safe evaluate xml body with bindings replaces bindings successfully" do + body = + "\n \n <{{date}} xmlns=\"{{key}}\">\n \n \n" + + bindings = %{ + date: "19700101", + key: "SECRET" + } + + expected = + "\n \n <19700101 xmlns=\"SECRET\">\n \n \n" + + assert UrlBuilder.safe_evaluate_body(body, bindings) == expected + end + + test "safe evaluate json body with bindings replaces bindings successfully" do + body = "{\"{{date}}\": \"{{key}}\"}" + + bindings = %{ + date: "19700101", + key: "SECRET" + } + + expected = "{\"19700101\": \"SECRET\"}" + + assert UrlBuilder.safe_evaluate_body(body, bindings) == expected + end + + test "safe evaluate body without bindings returns existing body with no changes" do + body = + "\n \n \n \n \n" + + bindings = %{} + + expected = + "\n \n \n \n \n" + + assert UrlBuilder.safe_evaluate_body(body, bindings) == expected + end end From 66385b3a3b6843a246c8c9be426f8cef413679ca Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Mon, 27 Mar 2023 16:47:43 -0500 Subject: [PATCH 02/11] version bump --- apps/andi/mix.exs | 2 +- apps/reaper/mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/andi/mix.exs b/apps/andi/mix.exs index a99f7c501..b0a18a321 100644 --- a/apps/andi/mix.exs +++ b/apps/andi/mix.exs @@ -4,7 +4,7 @@ defmodule Andi.MixProject do def project do [ app: :andi, - version: "2.6.3", + version: "2.6.4", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/reaper/mix.exs b/apps/reaper/mix.exs index eac3ad33a..dc9a77993 100644 --- a/apps/reaper/mix.exs +++ b/apps/reaper/mix.exs @@ -4,7 +4,7 @@ defmodule Reaper.MixProject do def project do [ app: :reaper, - version: "2.0.24", + version: "2.0.25", elixir: "~> 1.10", build_path: "../../_build", config_path: "../../config/config.exs", From bd0481487400fe143fba6b133ac6812c7ac2b580 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Mon, 27 Mar 2023 16:51:51 -0500 Subject: [PATCH 03/11] mix format --- apps/reaper/lib/reaper/http/downloader.ex | 1 + apps/reaper/test/unit/reaper/url_builder_test.exs | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/reaper/lib/reaper/http/downloader.ex b/apps/reaper/lib/reaper/http/downloader.ex index ef15fd1d9..ec67d039a 100644 --- a/apps/reaper/lib/reaper/http/downloader.ex +++ b/apps/reaper/lib/reaper/http/downloader.ex @@ -212,6 +212,7 @@ defmodule Reaper.Http.Downloader do end defp add_content_type(headers, ""), do: headers + defp add_content_type(headers, body) do case Jason.decode(body) do {:ok, _} -> diff --git a/apps/reaper/test/unit/reaper/url_builder_test.exs b/apps/reaper/test/unit/reaper/url_builder_test.exs index f67543a33..6d55d61b1 100644 --- a/apps/reaper/test/unit/reaper/url_builder_test.exs +++ b/apps/reaper/test/unit/reaper/url_builder_test.exs @@ -102,13 +102,11 @@ defmodule Reaper.UrlBuilderTest do end test "safe evaluate body without bindings returns existing body with no changes" do - body = - "\n \n \n \n \n" + body = "\n \n \n \n \n" bindings = %{} - expected = - "\n \n \n \n \n" + expected = "\n \n \n \n \n" assert UrlBuilder.safe_evaluate_body(body, bindings) == expected end From 6505a5f315cfab9d0ad6ea8ea29bc8258ecd25a8 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Tue, 28 Mar 2023 09:14:28 -0500 Subject: [PATCH 04/11] reaper test fixes --- .../test/unit/reaper/data_extract/processor_test.exs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/reaper/test/unit/reaper/data_extract/processor_test.exs b/apps/reaper/test/unit/reaper/data_extract/processor_test.exs index 83beb9e9e..ad3d9aeec 100644 --- a/apps/reaper/test/unit/reaper/data_extract/processor_test.exs +++ b/apps/reaper/test/unit/reaper/data_extract/processor_test.exs @@ -47,7 +47,7 @@ defmodule Reaper.DataExtract.ProcessorTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -162,7 +162,7 @@ defmodule Reaper.DataExtract.ProcessorTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -202,7 +202,7 @@ defmodule Reaper.DataExtract.ProcessorTest do context: %{ action: "GET", protocol: nil, - body: %{}, + body: "", url: sourceUrl, queryParams: %{}, headers: %{} @@ -256,7 +256,7 @@ defmodule Reaper.DataExtract.ProcessorTest do context: %{ action: "GET", protocol: nil, - body: %{}, + body: "", url: "#{sourceUrl}/{{currentYear}}-{{currentMonth}}", queryParams: %{}, headers: %{} @@ -356,7 +356,7 @@ defmodule Reaper.DataExtract.ProcessorTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], From 28fdf956f7c7a34a7d8520b0984015af7b3d701a Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Tue, 28 Mar 2023 09:28:20 -0500 Subject: [PATCH 05/11] reaper test fixes --- .../integration/reaper/reaper_full_test.exs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/reaper/test/integration/reaper/reaper_full_test.exs b/apps/reaper/test/integration/reaper/reaper_full_test.exs index d5c8d1d57..5e8d82f53 100644 --- a/apps/reaper/test/integration/reaper/reaper_full_test.exs +++ b/apps/reaper/test/integration/reaper/reaper_full_test.exs @@ -97,7 +97,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -177,7 +177,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -226,7 +226,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -261,7 +261,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -296,7 +296,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -330,7 +330,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -371,7 +371,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -433,7 +433,7 @@ defmodule Reaper.FullTest do context: %{ url: "http://localhost:#{bypass.port}/{{currentDate}}", action: "GET", - body: %{}, + body: "", protocol: nil, queryParams: %{}, headers: %{} @@ -569,7 +569,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -638,7 +638,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -687,7 +687,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -718,7 +718,7 @@ defmodule Reaper.FullTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], From 856cbedd065313c1c4e144e8bddac3d04373c832 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Tue, 28 Mar 2023 09:31:27 -0500 Subject: [PATCH 06/11] test fixes --- apps/e2e/test/e2e_test.exs | 4 ++-- apps/reaper/README.md | 2 +- .../reaper/test/integration/reaper/data_slurper/sftp_test.exs | 4 ++-- apps/reaper/test/integration/reaper/reaper_yeet_test.exs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/e2e/test/e2e_test.exs b/apps/e2e/test/e2e_test.exs index e17ea0f61..678a418d4 100644 --- a/apps/e2e/test/e2e_test.exs +++ b/apps/e2e/test/e2e_test.exs @@ -137,7 +137,7 @@ defmodule E2ETest do queryParams: %{}, headers: %{}, protocol: nil, - body: %{} + body: "" }, assigns: %{} } @@ -172,7 +172,7 @@ defmodule E2ETest do queryParams: %{}, headers: %{}, protocol: nil, - body: %{} + body: "" }, assigns: %{} } diff --git a/apps/reaper/README.md b/apps/reaper/README.md index 51d30b573..6738176f3 100644 --- a/apps/reaper/README.md +++ b/apps/reaper/README.md @@ -88,7 +88,7 @@ ingestion = %SmartCity.Ingestion{ assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], diff --git a/apps/reaper/test/integration/reaper/data_slurper/sftp_test.exs b/apps/reaper/test/integration/reaper/data_slurper/sftp_test.exs index a654a8db3..acebfb3cc 100644 --- a/apps/reaper/test/integration/reaper/data_slurper/sftp_test.exs +++ b/apps/reaper/test/integration/reaper/data_slurper/sftp_test.exs @@ -53,7 +53,7 @@ defmodule Reaper.SftpExtractorTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], @@ -99,7 +99,7 @@ defmodule Reaper.SftpExtractorTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], diff --git a/apps/reaper/test/integration/reaper/reaper_yeet_test.exs b/apps/reaper/test/integration/reaper/reaper_yeet_test.exs index 72c1f72d8..35f35d205 100644 --- a/apps/reaper/test/integration/reaper/reaper_yeet_test.exs +++ b/apps/reaper/test/integration/reaper/reaper_yeet_test.exs @@ -46,7 +46,7 @@ defmodule Reaper.YeetTest do assigns: %{}, context: %{ action: "GET", - body: %{}, + body: "", headers: [], protocol: nil, queryParams: [], From 556af1cfa5fc55cb915e009790d94d5ff5309c49 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Tue, 28 Mar 2023 15:29:13 -0500 Subject: [PATCH 07/11] bump smart_city_test and upgrade everywhere --- apps/alchemist/mix.exs | 2 +- apps/andi/82e2f93e-fbc7-45d8-b2b1-56a98e86e2bf | 1 + apps/andi/mix.exs | 2 +- apps/discovery_api/mix.exs | 2 +- apps/discovery_streams/mix.exs | 2 +- apps/e2e/test/e2e_test.exs | 4 +++- apps/estuary/mix.exs | 2 +- apps/flair/mix.exs | 2 +- apps/forklift/mix.exs | 2 +- apps/performance/mix.exs | 2 +- apps/pipeline/mix.exs | 2 +- apps/raptor/mix.exs | 2 +- apps/reaper/mix.exs | 2 +- apps/template/mix.exs | 2 +- apps/transformers/mix.exs | 2 +- apps/valkyrie/mix.exs | 2 +- mix.lock | 6 +++--- 17 files changed, 21 insertions(+), 18 deletions(-) create mode 100644 apps/andi/82e2f93e-fbc7-45d8-b2b1-56a98e86e2bf diff --git a/apps/alchemist/mix.exs b/apps/alchemist/mix.exs index eb61cb7e5..7d749428c 100644 --- a/apps/alchemist/mix.exs +++ b/apps/alchemist/mix.exs @@ -56,7 +56,7 @@ defmodule Alchemist.MixProject do {:properties, in_umbrella: true}, {:retry, "~> 0.13"}, {:smart_city, "~> 5.2.8", override: true}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:telemetry_event, in_umbrella: true}, {:timex, "~> 3.6"}, {:transformers, in_umbrella: true}, diff --git a/apps/andi/82e2f93e-fbc7-45d8-b2b1-56a98e86e2bf b/apps/andi/82e2f93e-fbc7-45d8-b2b1-56a98e86e2bf new file mode 100644 index 000000000..e7c7b8ce7 --- /dev/null +++ b/apps/andi/82e2f93e-fbc7-45d8-b2b1-56a98e86e2bf @@ -0,0 +1 @@ +true,foobar,10 \ No newline at end of file diff --git a/apps/andi/mix.exs b/apps/andi/mix.exs index b0a18a321..0c455d220 100644 --- a/apps/andi/mix.exs +++ b/apps/andi/mix.exs @@ -89,7 +89,7 @@ defmodule Andi.MixProject do {:raptor_service, in_umbrella: true}, {:simply_validate, ">= 0.2.0"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:sobelow, "~> 0.8", only: :dev}, {:ssl_verify_fun, "~> 1.1"}, {:sweet_xml, "~> 0.6"}, diff --git a/apps/discovery_api/mix.exs b/apps/discovery_api/mix.exs index 95b4cc4e4..40035880b 100644 --- a/apps/discovery_api/mix.exs +++ b/apps/discovery_api/mix.exs @@ -79,7 +79,7 @@ defmodule DiscoveryApi.Mixfile do {:redix, "~> 0.10"}, {:streaming_metrics, "~> 2.2"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:telemetry_event, in_umbrella: true}, {:temporary_env, "~> 2.0", only: :test, runtime: false}, {:timex, "~> 3.0"}, diff --git a/apps/discovery_streams/mix.exs b/apps/discovery_streams/mix.exs index b3feed31e..61dcc08c6 100644 --- a/apps/discovery_streams/mix.exs +++ b/apps/discovery_streams/mix.exs @@ -66,7 +66,7 @@ defmodule DiscoveryStreams.Mixfile do {:redix, "~> 0.10.2"}, {:sweet_xml, "~> 0.6"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:streaming_metrics, "~>2.1"}, {:telemetry_event, in_umbrella: true}, {:temporary_env, "~> 2.0", only: [:test, :integration]}, diff --git a/apps/e2e/test/e2e_test.exs b/apps/e2e/test/e2e_test.exs index 678a418d4..cf49fc87d 100644 --- a/apps/e2e/test/e2e_test.exs +++ b/apps/e2e/test/e2e_test.exs @@ -484,6 +484,7 @@ defmodule E2ETest do type: "auth", context: %{ destination: "dest", + body: "", url: "http://localhost:#{bypass.port()}/path/to/the/auth.json", path: ["token"], cacheTtl: 15_000 @@ -493,6 +494,7 @@ defmodule E2ETest do type: "http", context: %{ url: "http://localhost:#{bypass.port()}/path/to/the/data.csv", + body: "", action: "GET", headers: %{}, queryParams: %{} @@ -503,7 +505,7 @@ defmodule E2ETest do }) {:ok, andi_ingestion} = Andi.InputSchemas.Ingestions.update(smrt_ingestion) - + IO.inspect(andi_ingestion, label: "ingestion_test") ingestion_changeset = Andi.InputSchemas.InputConverter.andi_ingestion_to_full_ui_changeset_for_publish( andi_ingestion diff --git a/apps/estuary/mix.exs b/apps/estuary/mix.exs index 2e3907714..a751887c4 100644 --- a/apps/estuary/mix.exs +++ b/apps/estuary/mix.exs @@ -48,7 +48,7 @@ defmodule Estuary.MixProject do {:plug_heartbeat, "~> 0.2.0"}, {:prestige, "~> 2.0.0"}, {:properties, in_umbrella: true}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:sobelow, "~> 0.8", only: :dev}, {:quantum, "~>2.4"}, {:timex, "~> 3.6"} diff --git a/apps/flair/mix.exs b/apps/flair/mix.exs index e75968796..cdb4ca673 100644 --- a/apps/flair/mix.exs +++ b/apps/flair/mix.exs @@ -46,7 +46,7 @@ defmodule Flair.MixProject do {:placebo, "~> 2.0.0-rc2", only: [:dev, :test, :integration]}, {:faker, "~> 0.12", only: [:test, :integration], override: true}, {:mox, "~> 1.0", only: [:dev, :test, :integration]}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:distillery, "~> 2.1"}, {:pipeline, in_umbrella: true}, {:tasks, in_umbrella: true, only: :dev} diff --git a/apps/forklift/mix.exs b/apps/forklift/mix.exs index eb0049a3d..21aa4539f 100644 --- a/apps/forklift/mix.exs +++ b/apps/forklift/mix.exs @@ -53,7 +53,7 @@ defmodule Forklift.MixProject do {:redix, "~> 0.10"}, {:retry, "~> 0.14"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7"}, + {:smart_city_test, "~> 2.2.8"}, {:streaming_metrics, "~> 2.2"}, {:timex, "~> 3.6"}, {:distillery, "~> 2.1"}, diff --git a/apps/performance/mix.exs b/apps/performance/mix.exs index 2dc3b3142..4943a3080 100644 --- a/apps/performance/mix.exs +++ b/apps/performance/mix.exs @@ -24,7 +24,7 @@ defmodule Performance.MixProject do defp deps do [ {:combinatorics, "~> 0.1.0", only: [:integration]}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:benchee, "~> 1.1", only: [:integration]}, {:exprof, "~> 0.2.3", only: [:integration]}, {:retry, "~> 0.13"} diff --git a/apps/pipeline/mix.exs b/apps/pipeline/mix.exs index dd060edd9..6b307272e 100644 --- a/apps/pipeline/mix.exs +++ b/apps/pipeline/mix.exs @@ -44,7 +44,7 @@ defmodule Pipeline.MixProject do {:temp, "~> 0.4"}, {:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false}, {:placebo, "~> 2.0.0-rc2", only: [:dev, :test, :integration]}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:divo, "~> 1.3", only: [:dev, :integration]}, {:divo_kafka, "~> 0.1.5", only: [:dev, :integration]}, {:telemetry_event, in_umbrella: true} diff --git a/apps/raptor/mix.exs b/apps/raptor/mix.exs index 934a6deda..541d7e838 100644 --- a/apps/raptor/mix.exs +++ b/apps/raptor/mix.exs @@ -40,7 +40,7 @@ defmodule Raptor.MixProject do {:properties, in_umbrella: true}, {:redix, "~> 0.10"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:tasks, in_umbrella: true, only: :dev}, {:telemetry_event, in_umbrella: true}, {:tesla, "~> 1.3"}, diff --git a/apps/reaper/mix.exs b/apps/reaper/mix.exs index dc9a77993..c3fc95f6d 100644 --- a/apps/reaper/mix.exs +++ b/apps/reaper/mix.exs @@ -95,7 +95,7 @@ defmodule Reaper.MixProject do {:patiently, "~> 0.2", only: [:dev, :test, :integration], override: true}, {:phoenix, "~> 1.4", only: :test}, {:placebo, "~> 2.0.0-rc2", only: [:dev, :test, :integration]}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:temp, "~> 0.4", only: [:test, :integration]}, {:performance, in_umbrella: true, only: :integration}, {:unzip, "~> 0.6.0"} diff --git a/apps/template/mix.exs b/apps/template/mix.exs index 562ff6270..612a286b5 100644 --- a/apps/template/mix.exs +++ b/apps/template/mix.exs @@ -38,7 +38,7 @@ defmodule Template.MixProject do {:properties, in_umbrella: true}, {:redix, "~> 0.10"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:tasks, in_umbrella: true, only: :dev}, {:telemetry_event, in_umbrella: true}, {:distillery, "~> 2.1"} diff --git a/apps/transformers/mix.exs b/apps/transformers/mix.exs index 7a11ed1e2..ca7b46491 100644 --- a/apps/transformers/mix.exs +++ b/apps/transformers/mix.exs @@ -33,7 +33,7 @@ defmodule Transformers.MixProject do {:mox, "~> 1.0", only: [:dev, :test, :integration]}, {:placebo, "~> 2.0.0-rc2", only: [:dev, :test, :integration]}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:timex, "~> 3.6"}, {:decimal, "~> 1.0"} ] diff --git a/apps/valkyrie/mix.exs b/apps/valkyrie/mix.exs index 34bcda65f..748a425f5 100644 --- a/apps/valkyrie/mix.exs +++ b/apps/valkyrie/mix.exs @@ -56,7 +56,7 @@ defmodule Valkyrie.MixProject do {:properties, in_umbrella: true}, {:retry, "~> 0.13"}, {:smart_city, "~> 5.2.8"}, - {:smart_city_test, "~> 2.2.7", only: [:test, :integration]}, + {:smart_city_test, "~> 2.2.8", only: [:test, :integration]}, {:telemetry_event, in_umbrella: true}, {:timex, "~> 3.6"}, {:performance, in_umbrella: true, only: :integration} diff --git a/mix.lock b/mix.lock index a273fec51..e79eae157 100644 --- a/mix.lock +++ b/mix.lock @@ -40,7 +40,7 @@ "divo_postgres": {:hex, :divo_postgres, "0.2.0", "503b7047b1b577cd217e491fdfccd599b904963f1614a0ff24c2380cba5b69eb", [:mix], [{:divo, "~> 1.3", [hex: :divo, repo: "hexpm", optional: false]}], "hexpm", "1519487f4e6cbb6e359929934a0f3ba87374a0efd738c05a69c3ab44af86200f"}, "divo_redis": {:hex, :divo_redis, "0.1.4", "f5bbe82ce88bdd0ea54d6b14cd8dc1a800b872f0e488e6f2715b95cd6f349ee7", [:mix], [{:divo, "~> 1.3", [hex: :divo, repo: "hexpm", optional: false]}], "hexpm", "c85a10097a73b22a2ee1137af3af5c68bb5a6917c0147ffef3643ed82e41be46"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.29", "149d50dcb3a93d9f3d6f3ecf18c918fb5a2d3c001b5d3305c926cddfbd33355b", [:mix], [], "hexpm", "4902af1b3eb139016aed210888748db8070b8125c2342ce3dcae4f38dcc63503"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.31", "a93921cdc6b9b869f519213d5bc79d9e218ba768d7270d46fdcf1c01bacff9e2", [:mix], [], "hexpm", "317d367ee0335ef037a87e46c91a2269fef6306413f731e8ec11fc45a7efd059"}, "ecto": {:hex, :ecto, "3.9.0", "7c74fc0d950a700eb7019057ff32d047ed7f19b57c1b2ca260cf0e565829101d", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fed5ebc5831378b916afd0b5852a0c5bb3e7390665cc2b0ec8ab0c712495b73d"}, "ecto_sql": {:hex, :ecto_sql, "3.6.2", "9526b5f691701a5181427634c30655ac33d11e17e4069eff3ae1176c764e0ba3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.6.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ec9d7e6f742ea39b63aceaea9ac1d1773d574ea40df5a53ef8afbd9242fdb6b"}, "elastix": {:hex, :elastix, "0.8.0", "09b7ce7f0541bdd26225e29ade6360e3cf0c0e26088367b97b798e2f40e6c038", [:mix], [{:httpoison, "~> 1.4", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, "~> 4.0", [hex: :poison, repo: "hexpm", optional: true]}, {:retry, "~> 0.8", [hex: :retry, repo: "hexpm", optional: false]}], "hexpm", "8e7efc4bf6daf37bc99e2253a7f540c4e40464725a1ed3667a81de6ab5ed58d6"}, @@ -53,7 +53,7 @@ "ex_aws_cloudwatch": {:hex, :ex_aws_cloudwatch, "2.0.4", "f23ac70de91402e14b1393a349e1aba30478cf8af864687eeae05dc968f87acf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "faa5e5ba809c083b0460cc843b5082b200bb37d2abb25becb62c314b2796ae58"}, "ex_aws_s3": {:git, "https://github.com/ex-aws/ex_aws_s3", "6b9fdac73b62dee14bffb939965742f2576f2a7b", [ref: "6b9fdac73b62dee14bffb939965742f2576f2a7b"]}, "ex_aws_sts": {:hex, :ex_aws_sts, "2.3.0", "ce48c4cba7f1595a7d544458d0202ca313124026dba7b1a0021bbb1baa3d66d0", [:mix], [{:ex_aws, "~> 2.2", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "f14e4c7da3454514bf253b331e9422d25825485c211896ab3b81d2a4bdbf62f5"}, - "ex_doc": {:hex, :ex_doc, "0.29.1", "b1c652fa5f92ee9cf15c75271168027f92039b3877094290a75abcaac82a9f77", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "b7745fa6374a36daf484e2a2012274950e084815b936b1319aeebcf7809574f6"}, + "ex_doc": {:hex, :ex_doc, "0.29.3", "f07444bcafb302db86e4f02d8bbcd82f2e881a0dcf4f3e4740e4b8128b9353f7", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "3dc6787d7b08801ec3b51e9bd26be5e8826fbf1a17e92d1ebc252e1a1c75bfe1"}, "ex_json_schema": {:hex, :ex_json_schema, "0.7.3", "3289bf2edf57eb1ae0d5af35bc6d6c37d7e6d935f72e0120c7f0704510956049", [:mix], [], "hexpm", "d5389c44e2804d4e6cada6f4a99d68f9d2bc0e38c2e0fd21383f1878425bd5a9"}, "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e11a4490976aabeed3eb9dc70ec94a4f2d11fed5c9d4b5dc5d89bfa0a215abb5"}, "exprintf": {:hex, :exprintf, "0.2.1", "b7e895dfb00520cfb7fc1671303b63b37dc3897c59be7cbf1ae62f766a8a0314", [:mix], [], "hexpm", "20a0e8c880be90e56a77fcc82533c5d60c643915c7ce0cc8aa1e06ed6001da28"}, @@ -141,7 +141,7 @@ "simply_validate": {:hex, :simply_validate, "0.2.0", "d4b5b4a677ba70272bd55670ac216700a9704788b3074ac490cb02ee94f5b0c2", [:mix], [], "hexpm", "6c8bd42914d26710845ed567f5168b259590ea1067910c6565d03200edffe683"}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "smart_city": {:hex, :smart_city, "5.2.8", "262a090243b9922e8e29d5ee99342e6238a041872ad91ee08b26fd21d56c1e22", [:mix], [{:accessible, "~> 0.2.1", [hex: :accessible, repo: "hexpm", optional: false]}, {:brook_serializer, "~> 2.0", [hex: :brook_serializer, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.3", [hex: :mime, repo: "hexpm", optional: false]}, {:timex, "~> 3.6", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm", "d2a55a627c6aa72608f5409278937f8a240f73c3237e5e94532bfe7ee569f00a"}, - "smart_city_test": {:hex, :smart_city_test, "2.2.7", "af78708df869a47560d4ff8b2b3f5b7e53f2ab047c0bd87a37190c324b7d7b74", [:mix], [{:brod, "~> 3.8", [hex: :brod, repo: "hexpm", optional: false]}, {:ex_doc, "~> 0.19", [hex: :ex_doc, repo: "hexpm", optional: false]}, {:faker, "~> 0.12", [hex: :faker, repo: "hexpm", optional: false]}, {:gettext, "0.19.1", [hex: :gettext, repo: "hexpm", optional: false]}, {:patiently, "~> 0.2", [hex: :patiently, repo: "hexpm", optional: false]}, {:smart_city, "~> 5.2.8", [hex: :smart_city, repo: "hexpm", optional: false]}, {:timex, "3.7.8", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm", "0575977ada94504af0c1b58890401c4d18c53352a0bf77d277c975b9a3dac7a1"}, + "smart_city_test": {:hex, :smart_city_test, "2.2.8", "6cf4164d048a1a06da1d32dc4f18ef75e40797f54f85628f5176a8b6591c7eee", [:mix], [{:brod, "~> 3.8", [hex: :brod, repo: "hexpm", optional: false]}, {:ex_doc, "~> 0.19", [hex: :ex_doc, repo: "hexpm", optional: false]}, {:faker, "~> 0.12", [hex: :faker, repo: "hexpm", optional: false]}, {:gettext, "0.19.1", [hex: :gettext, repo: "hexpm", optional: false]}, {:patiently, "~> 0.2", [hex: :patiently, repo: "hexpm", optional: false]}, {:smart_city, "~> 5.2.8", [hex: :smart_city, repo: "hexpm", optional: false]}, {:timex, "3.7.8", [hex: :timex, repo: "hexpm", optional: false]}], "hexpm", "2865f94d5dfe1545823ef8ba8aa93f0eb3aa9d2ca3f1a65a944437f1c5637e56"}, "snappyer": {:hex, :snappyer, "1.2.5", "9154b9ac84031f0a799f72a4aa87df23ab2193b5631475fa2cdc304382d2df77", [:rebar3], [], "hexpm", "d2adc26a81efd5f138397a38a0bb545188d302972721f8be0de37fa452c8aed7"}, "sobelow": {:hex, :sobelow, "0.11.1", "23438964486f8112b41e743bbfd402da3e5b296fdc9eacab29914b79c48916dd", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "9897363a7eff96f4809304a90aad819e2ad5e5d24db547af502885146746a53c"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, From 2b37934b329d2967a30b2bb47ac560d0795ce3ad Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Tue, 28 Mar 2023 15:30:58 -0500 Subject: [PATCH 08/11] remove IO.inspect --- apps/e2e/13efb977-9b26-4a9b-afa7-c2143c00cf82 | 1 + apps/e2e/test/e2e_test.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 apps/e2e/13efb977-9b26-4a9b-afa7-c2143c00cf82 diff --git a/apps/e2e/13efb977-9b26-4a9b-afa7-c2143c00cf82 b/apps/e2e/13efb977-9b26-4a9b-afa7-c2143c00cf82 new file mode 100644 index 000000000..e7c7b8ce7 --- /dev/null +++ b/apps/e2e/13efb977-9b26-4a9b-afa7-c2143c00cf82 @@ -0,0 +1 @@ +true,foobar,10 \ No newline at end of file diff --git a/apps/e2e/test/e2e_test.exs b/apps/e2e/test/e2e_test.exs index cf49fc87d..53ab5c463 100644 --- a/apps/e2e/test/e2e_test.exs +++ b/apps/e2e/test/e2e_test.exs @@ -505,7 +505,7 @@ defmodule E2ETest do }) {:ok, andi_ingestion} = Andi.InputSchemas.Ingestions.update(smrt_ingestion) - IO.inspect(andi_ingestion, label: "ingestion_test") + ingestion_changeset = Andi.InputSchemas.InputConverter.andi_ingestion_to_full_ui_changeset_for_publish( andi_ingestion From 7ca9f1c59c60f68f116ea8f0c3db52f8446d7cab Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Tue, 28 Mar 2023 17:06:51 -0500 Subject: [PATCH 09/11] set initial body to empty string --- apps/andi/lib/andi/input_schemas/input_converter.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/andi/lib/andi/input_schemas/input_converter.ex b/apps/andi/lib/andi/input_schemas/input_converter.ex index 40626fff2..0a1e1274e 100644 --- a/apps/andi/lib/andi/input_schemas/input_converter.ex +++ b/apps/andi/lib/andi/input_schemas/input_converter.ex @@ -363,7 +363,7 @@ defmodule Andi.InputSchemas.InputConverter do defp update_context_from_andi_step(context, "http") do context |> cast_keys_to_atom_in_map_recursively() - |> Map.put_new(:body, %{}) + |> Map.put_new(:body, "") |> Map.put_new(:protocol, nil) |> Map.update(:queryParams, nil, &convert_key_value_to_map/1) |> Map.update(:headers, nil, &convert_key_value_to_map/1) @@ -419,7 +419,7 @@ defmodule Andi.InputSchemas.InputConverter do defp update_context_from_andi_step(context, "auth") do context - |> Map.put_new(:body, %{}) + |> Map.put_new(:body, "") |> Map.put_new(:encodeMethod, "json") |> Map.update(:headers, nil, &convert_key_value_to_map/1) end From 1c5d64341e3cc314159932af3d0c8645effbea14 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Wed, 29 Mar 2023 15:15:28 -0500 Subject: [PATCH 10/11] fix empty map body issue --- .../lib/reaper/data_extract/extract_step.ex | 2 +- apps/reaper/mix.exs | 2 +- .../reaper/data_extract/extract_step_test.exs | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/apps/reaper/lib/reaper/data_extract/extract_step.ex b/apps/reaper/lib/reaper/data_extract/extract_step.ex index 7a19a1dd2..e2896021d 100644 --- a/apps/reaper/lib/reaper/data_extract/extract_step.ex +++ b/apps/reaper/lib/reaper/data_extract/extract_step.ex @@ -97,7 +97,7 @@ defmodule Reaper.DataExtract.ExtractStep do {body, headers} end - defp process_body(body, _assigns) when body in ["", nil], do: "" + defp process_body(body, _assigns) when body in ["", nil, %{}], do: "" defp process_body(body, assigns) do body |> UrlBuilder.safe_evaluate_body(assigns) diff --git a/apps/reaper/mix.exs b/apps/reaper/mix.exs index c3fc95f6d..544b22eeb 100644 --- a/apps/reaper/mix.exs +++ b/apps/reaper/mix.exs @@ -4,7 +4,7 @@ defmodule Reaper.MixProject do def project do [ app: :reaper, - version: "2.0.25", + version: "2.0.26", elixir: "~> 1.10", build_path: "../../_build", config_path: "../../config/config.exs", diff --git a/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs b/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs index ddbc282d2..13d7e5dd1 100644 --- a/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs +++ b/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs @@ -230,6 +230,39 @@ defmodule Reaper.DataExtract.ExtractStepTest do assert assigns == %{key: "super secret two", token: "auth_token2"} end + test "Can use empty map for body", %{bypass: bypass, ingestion: ingestion} do + Bypass.stub(bypass, "POST", "/", fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + + case body do + "" -> Plug.Conn.resp(conn, 200, %{sub: %{path: "auth_token2"}} |> Jason.encode!()) + _ -> Plug.Conn.resp(conn, 403, "No dice") + end + end) + + steps = [ + %{ + type: "auth", + context: %{ + path: ["sub", "path"], + destination: "token", + url: "http://localhost:#{bypass.port}", + encodeMethod: "json", + body: %{}, + headers: %{}, + cacheTtl: nil + }, + assigns: %{ + key: "super secret two" + } + } + ] + + assigns = ExtractStep.execute_extract_steps(ingestion, steps) + + assert assigns == %{key: "super secret two", token: "auth_token2"} + end + test "Can use assigns block for headers", %{bypass: bypass, ingestion: ingestion} do Bypass.stub(bypass, "POST", "/headers", fn conn -> if Enum.any?(conn.req_headers, fn header -> header == {"header", "super secret"} end) do From 157ec9663c3aee940bc521678de86f0cda31363c Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Thu, 30 Mar 2023 09:02:08 -0500 Subject: [PATCH 11/11] add empty list handling to body --- .../lib/reaper/data_extract/extract_step.ex | 2 +- apps/reaper/mix.exs | 2 +- .../reaper/data_extract/extract_step_test.exs | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/apps/reaper/lib/reaper/data_extract/extract_step.ex b/apps/reaper/lib/reaper/data_extract/extract_step.ex index e2896021d..df8bfc247 100644 --- a/apps/reaper/lib/reaper/data_extract/extract_step.ex +++ b/apps/reaper/lib/reaper/data_extract/extract_step.ex @@ -97,7 +97,7 @@ defmodule Reaper.DataExtract.ExtractStep do {body, headers} end - defp process_body(body, _assigns) when body in ["", nil, %{}], do: "" + defp process_body(body, _assigns) when body in ["", nil, %{}, []], do: "" defp process_body(body, assigns) do body |> UrlBuilder.safe_evaluate_body(assigns) diff --git a/apps/reaper/mix.exs b/apps/reaper/mix.exs index 544b22eeb..e3aa41e61 100644 --- a/apps/reaper/mix.exs +++ b/apps/reaper/mix.exs @@ -4,7 +4,7 @@ defmodule Reaper.MixProject do def project do [ app: :reaper, - version: "2.0.26", + version: "2.0.27", elixir: "~> 1.10", build_path: "../../_build", config_path: "../../config/config.exs", diff --git a/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs b/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs index 13d7e5dd1..d4aa2c8a8 100644 --- a/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs +++ b/apps/reaper/test/unit/reaper/data_extract/extract_step_test.exs @@ -263,6 +263,39 @@ defmodule Reaper.DataExtract.ExtractStepTest do assert assigns == %{key: "super secret two", token: "auth_token2"} end + test "Can use empty list for body", %{bypass: bypass, ingestion: ingestion} do + Bypass.stub(bypass, "POST", "/", fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + + case body do + "" -> Plug.Conn.resp(conn, 200, %{sub: %{path: "auth_token2"}} |> Jason.encode!()) + _ -> Plug.Conn.resp(conn, 403, "No dice") + end + end) + + steps = [ + %{ + type: "auth", + context: %{ + path: ["sub", "path"], + destination: "token", + url: "http://localhost:#{bypass.port}", + encodeMethod: "json", + body: [], + headers: %{}, + cacheTtl: nil + }, + assigns: %{ + key: "super secret two" + } + } + ] + + assigns = ExtractStep.execute_extract_steps(ingestion, steps) + + assert assigns == %{key: "super secret two", token: "auth_token2"} + end + test "Can use assigns block for headers", %{bypass: bypass, ingestion: ingestion} do Bypass.stub(bypass, "POST", "/headers", fn conn -> if Enum.any?(conn.req_headers, fn header -> header == {"header", "super secret"} end) do