diff --git a/apps/andi/assets/css/ingestions.scss b/apps/andi/assets/css/ingestions.scss index 93343fee3..50d06e890 100644 --- a/apps/andi/assets/css/ingestions.scss +++ b/apps/andi/assets/css/ingestions.scss @@ -4,6 +4,7 @@ .ingestion-metadata-form { margin-bottom: 1rem; + margin-top: 1rem; } .selected-results-from-search.selected-dataset { diff --git a/apps/andi/lib/andi_web/live/helpers/metadata_form_helpers.ex b/apps/andi/lib/andi_web/live/helpers/metadata_form_helpers.ex index ff858899a..bfb92429f 100644 --- a/apps/andi/lib/andi_web/live/helpers/metadata_form_helpers.ex +++ b/apps/andi/lib/andi_web/live/helpers/metadata_form_helpers.ex @@ -28,6 +28,7 @@ defmodule AndiWeb.Helpers.MetadataFormHelpers do end def get_source_format_options(_), do: Options.source_format() + def get_source_format_options(), do: Options.source_format() def get_language(nil), do: "english" def get_language(lang), do: lang diff --git a/apps/andi/lib/andi_web/live/ingestion_live_view/data_dictionary_form.ex b/apps/andi/lib/andi_web/live/ingestion_live_view/data_dictionary_form.ex index f16a76f5a..2b5f27289 100644 --- a/apps/andi/lib/andi_web/live/ingestion_live_view/data_dictionary_form.ex +++ b/apps/andi/lib/andi_web/live/ingestion_live_view/data_dictionary_form.ex @@ -51,6 +51,12 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryForm do "expanded" -> "MINIMIZE" end + loader_visibility = + case assigns.loading_schema do + true -> "loading" + false -> "hidden" + end + ~L"""
@@ -73,10 +79,27 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryForm do
- <%= ErrorHelpers.error_tag(f, :schema, bind_to_input: false, class: "full-width") %> -
+
+ <%= if @sourceFormat in ["text/csv", "application/json"] and @is_curator do %> +
+
+
+ <%= label(f, :schema_sample, "Upload data sample", class: "label") %> + <%= file_input(f, :schema_sample, phx_hook: "readFile", accept: "text/csv, application/json") %> + <%= ErrorHelpers.error_tag(f, :schema_sample, bind_to_input: false) %> +
+
+ + +
+
+ <% end %> + + <%= ErrorHelpers.error_tag(f, :schema, bind_to_input: false, class: "full-width") %> +
+
Enter/Edit Fields
@@ -105,6 +128,7 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryForm do <%= live_component(@socket, AndiWeb.DataDictionary.RemoveFieldEditor, id: :data_dictionary_remove_field_editor, selected_field: @current_data_dictionary_item, visible: @remove_data_dictionary_field_visible) %> + <%= live_component(@socket, AndiWeb.DataDictionary.OverwriteSchemaModal, id: :overwrite_schema_modal, visibility: @overwrite_schema_visibility) %>
""" end @@ -496,9 +520,4 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryForm do _ -> {:noreply, assign(socket, changeset: new_changeset, loading_schema: false)} end end - - defp send_data_dictionary_status(changeset, socket) do - send(socket.parent_pid, {:update_data_dictionary_status, changeset.valid?}) - changeset - end end diff --git a/apps/andi/lib/andi_web/live/ingestion_live_view/edit_ingestion_live_view.ex b/apps/andi/lib/andi_web/live/ingestion_live_view/edit_ingestion_live_view.ex index 8eaa396fb..1865da8dc 100644 --- a/apps/andi/lib/andi_web/live/ingestion_live_view/edit_ingestion_live_view.ex +++ b/apps/andi/lib/andi_web/live/ingestion_live_view/edit_ingestion_live_view.ex @@ -14,27 +14,30 @@ defmodule AndiWeb.IngestionLiveView.EditIngestionLiveView do ~L""" <%= header_render(@socket, @is_curator) %>
-
-

Define Data Ingestion

-
- -
- <%= live_render(@socket, AndiWeb.IngestionLiveView.MetadataForm, id: :ingestion_metadata_form_editor, session: %{"ingestion" => @ingestion}) %> -
- -
-
- <%= live_render(@socket, AndiWeb.IngestionLiveView.ExtractSteps.ExtractStepForm, id: :extract_step_form_editor, session: %{"ingestion" => @ingestion, "order" => "1"}) %> + <%= f = form_for @changeset, "" %> + <%= hidden_input(f, :sourceFormat) %> +
+

Define Data Ingestion

-
- <%= live_render(@socket, AndiWeb.IngestionLiveView.DataDictionaryForm, id: :data_dictionary_form_editor, session: %{"ingestion" => @ingestion, "is_curator" => @is_curator, "order" => "2"}) %> +
+ <%= live_render(@socket, AndiWeb.IngestionLiveView.MetadataForm, id: :ingestion_metadata_form_editor, session: %{"ingestion" => @ingestion}) %>
-
- <%= live_render(@socket, AndiWeb.IngestionLiveView.FinalizeForm, id: :finalize_form_editor, session: %{"ingestion" => @ingestion, "order" => "4"}) %> +
+
+ <%= live_render(@socket, AndiWeb.IngestionLiveView.ExtractSteps.ExtractStepForm, id: :extract_step_form_editor, session: %{"ingestion" => @ingestion, "order" => "1"}) %> +
+ +
+ <%= live_render(@socket, AndiWeb.IngestionLiveView.DataDictionaryForm, id: :data_dictionary_form_editor, session: %{"ingestion" => @ingestion, "is_curator" => @is_curator, "order" => "2"}) %> +
+ +
+ <%= live_render(@socket, AndiWeb.IngestionLiveView.FinalizeForm, id: :finalize_form_editor, session: %{"ingestion" => @ingestion, "order" => "4"}) %> +
-
+
diff --git a/apps/andi/lib/andi_web/live/ingestion_live_view/metadata_form.ex b/apps/andi/lib/andi_web/live/ingestion_live_view/metadata_form.ex index 404ec9fa6..b63607c7a 100644 --- a/apps/andi/lib/andi_web/live/ingestion_live_view/metadata_form.ex +++ b/apps/andi/lib/andi_web/live/ingestion_live_view/metadata_form.ex @@ -13,6 +13,7 @@ defmodule AndiWeb.IngestionLiveView.MetadataForm do def mount(_, %{"ingestion" => ingestion}, socket) do changeset = IngestionMetadataFormSchema.changeset_from_andi_ingestion(ingestion) AndiWeb.Endpoint.subscribe("form-save") + AndiWeb.Endpoint.subscribe("source-format") {:ok, assign(socket, @@ -20,7 +21,8 @@ defmodule AndiWeb.IngestionLiveView.MetadataForm do select_dataset_modal_visibility: "hidden", search_results: [], search_text: "", - selected_dataset: ingestion.targetDataset + selected_dataset: ingestion.targetDataset, + ingestion_id: ingestion.id )} end @@ -34,7 +36,7 @@ defmodule AndiWeb.IngestionLiveView.MetadataForm do
<%= label(f, :sourceFormat, "Source Format", class: "label label--required") %> - <%= select(f, :sourceFormat, MetadataFormHelpers.get_source_format_options(input_value(f, :sourceFormat)), [class: "select ingestion-form-fields"]) %> + <%= select(f, :sourceFormat, MetadataFormHelpers.get_source_format_options(), [class: "select ingestion-form-fields"]) %> <%= ErrorHelpers.error_tag(f, :sourceFormat, bind_to_input: false) %>
@@ -78,6 +80,17 @@ defmodule AndiWeb.IngestionLiveView.MetadataForm do {:noreply, assign(socket, selected_dataset: nil)} end + def handle_event("validate", %{"form_data" => form_data, "_target" => ["form_data", "sourceFormat"]}, socket) do + AndiWeb.Endpoint.broadcast_from(self(), "source-format", "format-update", %{ + new_format: form_data["sourceFormat"], + ingestion_id: socket.assigns.ingestion_id + }) + + form_data + |> IngestionMetadataFormSchema.changeset_from_form_data() + |> complete_validation(socket) + end + def handle_event("validate", %{"form_data" => form_data}, socket) do form_data |> IngestionMetadataFormSchema.changeset_from_form_data() diff --git a/apps/andi/mix.exs b/apps/andi/mix.exs index ac2762fdc..e76a7c87f 100644 --- a/apps/andi/mix.exs +++ b/apps/andi/mix.exs @@ -4,7 +4,7 @@ defmodule Andi.MixProject do def project do [ app: :andi, - version: "2.2.8", + version: "2.2.9", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/andi/test/integration/andi_web/live/ingestion_live_view/data_dictionary_form_test.exs b/apps/andi/test/integration/andi_web/live/ingestion_live_view/data_dictionary_form_test.exs index c621352d9..18d9f3d4e 100644 --- a/apps/andi/test/integration/andi_web/live/ingestion_live_view/data_dictionary_form_test.exs +++ b/apps/andi/test/integration/andi_web/live/ingestion_live_view/data_dictionary_form_test.exs @@ -6,6 +6,7 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryFormTest do import SmartCity.Event, only: [ingestion_update: 0, dataset_update: 0] import SmartCity.TestHelper, only: [eventually: 1, eventually: 3] import Phoenix.LiveViewTest + import Checkov alias SmartCity.TestDataGenerator, as: TDG alias Andi.InputSchemas.DataDictionaryFields @@ -895,6 +896,248 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryFormTest do end end + describe "schema sample upload" do + test "is shown when sourceFormat is CSV or JSON", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "application/json", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + html = render(data_dictionary_view) + + refute Enum.empty?(find_elements(html, ".data-dictionary-form__file-upload")) + end + + test "is hidden when sourceFormat is not CSV nor JSON", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "application/geo+json", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + html = render(data_dictionary_view) + + assert Enum.empty?(find_elements(html, ".data-dictionary-form__file-upload")) + end + + test "does not allow file uploads greater than 200MB", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "text/csv", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 200_000_001}) + + refute Enum.empty?(find_elements(html, "#schema_sample-error-msg")) + end + + test "should throw error when empty csv file is passed", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "text/csv", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = "" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "text/csv", "file" => csv_sample}) + + refute Enum.empty?(find_elements(html, "#schema_sample-error-msg")) + end + + data_test "accepts common csv file type #{type}", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "text/csv", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = "string,int,float,bool,date\nabc,9,1.5,true,2020-07-22T21:24:40" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 10, "fileType" => type, "file" => csv_sample}) + + assert Enum.empty?(find_elements(html, "#schema_sample-error-msg")) + + where([ + [:type], + ["text/csv"], + ["application/vnd.ms-excel"] + ]) + end + + test "should throw error when empty csv file with `\n` is passed", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "text/csv", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = "\n" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "text/csv", "file" => csv_sample}) + + refute Enum.empty?(find_elements(html, "#schema_sample-error-msg")) + end + + test "provides modal when existing schema will be overwritten", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + ingestion = TDG.create_ingestion(%{sourceFormat: "text/csv", targetDataset: dataset.id}) + + {:ok, _} = Datasets.update(dataset) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = "CAM\nrules" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "text/csv", "file" => csv_sample}) + + refute Enum.empty?(find_elements(html, ".overwrite-schema-modal--visible")) + end + + test "does not provide modal with no existing schema", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + {:ok, _} = Datasets.update(dataset) + ingestion = Ingestions.create(dataset.id) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + select_source_format("text/csv", view) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = "CAM\nrules" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "text/csv", "file" => csv_sample}) + + assert Enum.empty?(find_elements(html, ".overwrite-schema-modal--visible")) + + updated_ingestion = Ingestions.get(ingestion.id) + refute Enum.empty?(updated_ingestion.schema) + end + + test "parses CSVs with various types", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + {:ok, _} = Datasets.update(dataset) + ingestion = Ingestions.create(dataset.id) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + select_source_format("text/csv", view) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = "string,int,float,bool,date,timestamp\nabc,9,1.5,true,2020-07-22,2020-07-22T21:24:40" + + render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "text/csv", "file" => csv_sample}) + + updated_ingestion = Ingestions.get(ingestion.id) + + generated_schema = + updated_ingestion.schema + |> Enum.map(fn item -> %{type: item.type, name: item.name} end) + + expected_schema = [ + %{name: "string", type: "string"}, + %{name: "int", type: "integer"}, + %{name: "float", type: "float"}, + %{name: "bool", type: "boolean"}, + %{name: "date", type: "date"}, + %{name: "timestamp", type: "timestamp"} + ] + + assert generated_schema == expected_schema + end + + test "parses CSV with valid column names", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + {:ok, _} = Datasets.update(dataset) + ingestion = Ingestions.create(dataset.id) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + select_source_format("text/csv", view) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + csv_sample = + "string\r,i&^%$nt,fl\toat,bool---,date as multi word column,timestamp as multi word column\nabc,9,1.5,true,2020-07-22,2020-07-22T21:24:40" + + render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "text/csv", "file" => csv_sample}) + + updated_ingestion = Ingestions.get(ingestion.id) + + generated_schema = + updated_ingestion.schema + |> Enum.map(fn item -> %{type: item.type, name: item.name} end) + + expected_schema = [ + %{name: "string", type: "string"}, + %{name: "int", type: "integer"}, + %{name: "float", type: "float"}, + %{name: "bool", type: "boolean"}, + %{name: "date as multi word column", type: "date"}, + %{name: "timestamp as multi word column", type: "timestamp"} + ] + + assert generated_schema == expected_schema + end + + test "handles invalid json", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + {:ok, _} = Datasets.update(dataset) + ingestion = TDG.create_ingestion(%{sourceFormat: "application/json", targetDataset: dataset.id}) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + json_sample = "header" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "application/json", "file" => json_sample}) + + refute Enum.empty?(find_elements(html, "#schema_sample-error-msg")) + end + + test "should throw error when empty json file is passed", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + {:ok, _} = Datasets.update(dataset) + ingestion = TDG.create_ingestion(%{sourceFormat: "application/json", targetDataset: dataset.id}) + {:ok, _} = Ingestions.update(ingestion) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + json_sample = "[]" + + html = render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "application/json", "file" => json_sample}) + refute Enum.empty?(find_elements(html, "#schema_sample-error-msg")) + end + + test "generates eligible parents list", %{conn: conn} do + dataset = TDG.create_dataset(%{}) + {:ok, _} = Datasets.update(dataset) + ingestion = Ingestions.create(dataset.id) + assert {:ok, view, html} = live(conn, @url_path <> ingestion.id) + select_source_format("application/json", view) + data_dictionary_view = find_live_child(view, "data_dictionary_form_editor") + + json_sample = [%{list_field: [%{child_list_field: []}]}] |> Jason.encode!() + + render_hook(data_dictionary_view, "file_upload", %{"fileSize" => 100, "fileType" => "application/json", "file" => json_sample}) + + updated_ingestion = Ingestions.get(ingestion.id) + + generated_bread_crumbs = + updated_ingestion + |> DataDictionaryFields.get_parent_ids_from_ingestion() + |> Enum.map(fn {bread_crumb, _} -> bread_crumb end) + + assert ["Top Level", "list_field", "list_field > child_list_field"] == generated_bread_crumbs + end + end + defp create_ingestion_with_schema(schema) do dataset = TDG.create_dataset(%{}) ingestion = TDG.create_ingestion(%{targetDataset: dataset.id, schema: schema}) @@ -908,4 +1151,16 @@ defmodule AndiWeb.IngestionLiveView.DataDictionaryFormTest do Ingestions.get(ingestion.id) end + + defp select_source_format(source_format, view) do + new_source_format = source_format + + form_data = %{ + "sourceFormat" => new_source_format + } + + metadata_view = find_live_child(view, "ingestion_metadata_form_editor") + render_change(metadata_view, "validate", %{"form_data" => form_data}) + render_change(view, "save") + end end