Trino upgrade

apps/andi/config/integration.exs

@@ -1,5 +1,4 @@
 use Mix.Config
-import_config "../test/integration/divo_minio.ex"
 
 System.put_env("AUTH0_DOMAIN", "project-hercules.us.auth0.com")
 System.put_env("AUTH0_CLIENT_ID", "VHr6xrLKUMsLg1AZYXXLgJBI3LOhcLbY")
@@ -18,22 +17,10 @@ db_username = "postgres"
 db_password = "postgres"
 db_port = "5456"
 
-bucket_name = "kdp-cloud-storage"
+bucket_name = "trino-hive-storage"
 
 config :andi,
-  divo: [
-    {DivoKafka, [create_topics: "event-stream:1:1", outside_host: host, kafka_image_version: "2.12-2.1.1"]},
-    {DivoRedis, []},
-    {
-      DivoPostgres,
-      [
-        user: db_username,
-        database: db_name,
-        port: db_port
-      ]
-    },
-    {Andi.DivoMinio, [bucket_name: bucket_name]}
-  ],
+  divo: "docker-compose.yml",
   divo_wait: [dwell: 700, max_tries: 50],
   kafka_broker: endpoint,
   dead_letter_topic: "dead-letters",
@@ -153,8 +140,8 @@ config :andi, Guardian.DB, repo: Andi.Repo
 
 config :ex_aws,
   debug_requests: true,
-  access_key_id: "testing_access_key",
-  secret_access_key: "testing_secret_key",
+  access_key_id: "minioadmin",
+  secret_access_key: "minioadmin",
   region: "local"
 
 config :ex_aws, :s3,
@@ -165,5 +152,5 @@ config :ex_aws, :s3,
   },
   port: 9000
 
-System.put_env("AWS_ACCESS_KEY_ID", "testing_access_key")
-System.put_env("AWS_ACCESS_KEY_SECRET", "testing_secret_key")
+System.put_env("AWS_ACCESS_KEY_ID", "minioadmin")
+System.put_env("AWS_ACCESS_KEY_SECRET", "minioadmin")

apps/andi/config/test.exs

@@ -34,5 +34,5 @@ config :andi, AndiWeb.Auth.TokenHandler,
 
 config :andi, Guardian.DB, repo: Andi.Repo
 
-System.put_env("AWS_ACCESS_KEY_ID", "testing_access_key")
-System.put_env("AWS_ACCESS_KEY_SECRET", "testing_secret_key")
+System.put_env("AWS_ACCESS_KEY_ID", "minioadmin")
+System.put_env("AWS_ACCESS_KEY_SECRET", "minioadmin")

apps/andi/docker-compose.yml

@@ -0,0 +1,56 @@
+---
+version: '3.4'
+services:
+  init:
+    image: alpine:3.16.0
+    volumes:
+      - ${PWD}/test/integration:/workdir
+      - minio:/minio
+    entrypoint: /workdir/setup.sh
+  minio:
+    image: minio/minio:RELEASE.2022-07-17T15-43-14Z.fips
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio:/data
+    ports:
+    - "9000:9000"
+    - "9001:9001"
+    depends_on:
+      - init
+  ecto-postgres:
+    image: postgres:14.4-alpine
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: andi_test
+    ports:
+      - "5456:5432"
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+    - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:2.12-2.1.1
+    depends_on:
+    - zookeeper
+    ports:
+    - "9092:9092"
+    environment:
+    - "KAFKA_ADVERTISED_LISTENERS=INSIDE://:9094,OUTSIDE://${HOST_IP:-localhost}:9092"
+    - "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
+    - "KAFKA_LISTENERS=INSIDE://:9094,OUTSIDE://:9092"
+    - "KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE"
+    - "KAFKA_CREATE_TOPICS=event-stream:1:1,streaming-transformed:1:1,dead-letters:1:1,streaming-persisted:1:1"
+    - "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181"
+    healthcheck:
+      test: ["CMD-SHELL", "cat /opt/kafka/logs/server.log | grep \"Previous Leader Epoch was: -1\""]
+      interval: 10s
+      timeout: 20s
+      retries: 3
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"
+
+volumes:
+  minio:

apps/andi/mix.exs

@@ -4,7 +4,7 @@ defmodule Andi.MixProject do
   def project do
     [
       app: :andi,
-      version: "2.2.20",
+      version: "2.3.0",
       build_path: "../../_build",
       config_path: "../../config/config.exs",
       deps_path: "../../deps",
@@ -79,7 +79,7 @@ defmodule Andi.MixProject do
       {:placebo, "~> 2.0.0-rc2", only: [:dev, :test, :integration]},
       {:plug_cowboy, "~> 2.5"},
       {:postgrex, "~> 0.15.1"},
-      {:prestige, "~> 1.0"},
+      {:prestige, "~> 2.0.0"},
       {:properties, in_umbrella: true},
       {:quantum, "~> 2.4"},
       {:ranch, "~> 1.7.1", override: true},

apps/andi/test/integration/andi_web/live/edit_user_live_view_test.exs

@@ -116,8 +116,10 @@ defmodule AndiWeb.EditUserLiveViewTest do
       org_id = org.id
 
       eventually(fn ->
-        assert [{"Please select an organization", ""}, {"Awesome Title", org_id} | rest] =
-                 get_all_select_options(html, "#organiation_org_id")
+        assert {:ok, view, html} = live(conn, @url_path <> user.id)
+        options = get_all_select_options(html, "#organiation_org_id")
+        assert {"Please select an organization", ""} == List.first(options)
+        assert {"Awesome Title", org_id} in options
       end)
 
       render_change(view, "associate", %{"organiation" => %{"org_id" => org_id}})

apps/andi/test/integration/andi_web/live/submission_upload_data_dictionary_test.exs

@@ -67,7 +67,7 @@ defmodule AndiWeb.SubmissionUploadDataDictionaryTest do
 
       eventually(fn ->
         assert {:ok, _} =
-                 ExAws.S3.get_object("kdp-cloud-storage", "samples/#{andi_dataset.id}/sample.json")
+                 ExAws.S3.get_object("trino-hive-storage", "samples/#{andi_dataset.id}/sample.json")
                  |> ExAws.request()
 
         assert "sample.json" == get_text(html, ".sample-file-display")
@@ -90,7 +90,7 @@ defmodule AndiWeb.SubmissionUploadDataDictionaryTest do
 
       eventually(fn ->
         assert {:ok, _} =
-                 ExAws.S3.get_object("kdp-cloud-storage", "samples/#{andi_dataset.id}/sample.csv")
+                 ExAws.S3.get_object("trino-hive-storage", "samples/#{andi_dataset.id}/sample.csv")
                  |> ExAws.request()
 
         assert "sample.csv" == get_text(html, ".sample-file-display")

apps/andi/test/integration/hive.properties

@@ -0,0 +1,11 @@
+connector.name=hive-hadoop2
+hive.metastore.uri=thrift://metastore:8000
+hive.metastore.username=hive
+hive.metastore-timeout=360m
+hive.allow-drop-table=true
+hive.allow-rename-table=true
+hive.s3.aws-access-key=minioadmin
+hive.s3.aws-secret-key=minioadmin
+hive.s3.path-style-access=true
+hive.s3.endpoint=http://minio:9000
+hive.s3.ssl.enabled=false

apps/andi/test/integration/setup.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+echo "Beginning setup"
+rm -rf /minio/trino-hive-storage || true
+mkdir /minio/trino-hive-storage
+echo "Setup complete"
+ls minio

apps/discovery_api/config/integration.exs

@@ -1,7 +1,7 @@
 use Mix.Config
 aws_region = "local"
-System.put_env("AWS_ACCESS_KEY_ID", "testing_access_key")
-System.put_env("AWS_ACCESS_KEY_SECRET", "testing_secret_key")
+System.put_env("AWS_ACCESS_KEY_ID", "minioadmin")
+System.put_env("AWS_ACCESS_KEY_SECRET", "minioadmin")
 host = "localhost"
 endpoints = [{to_charlist(host), 9092}]
 redix_args = [host: host]
@@ -14,7 +14,7 @@ config :discovery_api,
   allowed_origins: ["integrationtests.example.com", "localhost:9001"],
   divo: "test/integration/docker-compose.yaml",
   divo_wait: [dwell: 2000, max_tries: 35],
-  hosted_bucket: "kdp-cloud-storage",
+  hosted_bucket: "trino-hive-storage",
   hosted_region: aws_region,
   hsts_enabled: false
 

apps/discovery_api/config/test.exs

@@ -1,7 +1,7 @@
 use Mix.Config
 
-System.put_env("AWS_ACCESS_KEY_ID", "testing_access_key")
-System.put_env("AWS_ACCESS_KEY_SECRET", "testing_secret_key")
+System.put_env("AWS_ACCESS_KEY_ID", "minioadmin")
+System.put_env("AWS_ACCESS_KEY_SECRET", "minioadmin")
 config :prestige, :session_opts, url: "http://localhost:8080"
 host = "localhost"
 

apps/discovery_api/mix.exs

@@ -5,7 +5,7 @@ defmodule DiscoveryApi.Mixfile do
     [
       app: :discovery_api,
       compilers: [:phoenix, :gettext | Mix.compilers()],
-      version: "1.2.17",
+      version: "1.3.0",
       build_path: "../../_build",
       config_path: "../../config/config.exs",
       deps_path: "../../deps",
@@ -70,7 +70,7 @@ defmodule DiscoveryApi.Mixfile do
       {:placebo, "~> 2.0.0-rc2", only: [:dev, :test, :integration]},
       {:plug_heartbeat, "~> 0.2.0"},
       {:postgrex, "~> 0.15.1"},
-      {:prestige, "~> 1.0"},
+      {:prestige, "~> 2.0.0"},
       {:properties, in_umbrella: true},
       {:quantum, "~>2.4"},
       {:ranch, "~> 1.7.1", override: true},

apps/discovery_api/test/integration/discovery_api/data/hosted_file_test.exs

@@ -23,8 +23,8 @@ defmodule DiscoveryApi.Data.HostedFileTest do
   setup do
     allow(RaptorService.list_access_groups_by_dataset(any(), any()), return: %{access_groups: []})
 
-    Application.put_env(:ex_aws, :access_key_id, "testing_access_key")
-    Application.put_env(:ex_aws, :secret_access_key, "testing_secret_key")
+    Application.put_env(:ex_aws, :access_key_id, "minioadmin")
+    Application.put_env(:ex_aws, :secret_access_key, "minioadmin")
 
     Redix.command!(:redix, ["FLUSHALL"])
 

apps/discovery_api/test/integration/discovery_api/data/query_test.exs

@@ -311,7 +311,7 @@ defmodule DiscoveryApi.Data.QueryTest do
 
       assert %{
                "message" =>
-                 "Syntax Error: mismatched input 'FORM'. Expecting: ',', 'EXCEPT', 'FROM', 'GROUP', 'HAVING', 'INTERSECT', 'LIMIT', 'ORDER', 'UNION', 'WHERE', <EOF>"
+                 "Syntax Error: Invalid X-Trino-Prepared-Statement header: line 1:43: mismatched input 'FORM'. Expecting: ',', 'EXCEPT', 'FETCH', 'FROM', 'GROUP', 'HAVING', 'INTERSECT', 'LIMIT', 'OFFSET', 'ORDER', 'UNION', 'WHERE', 'WINDOW', <EOF>"
              } ==
                plain_text_post(conn, "/api/v1/query", request_body)
                |> response(400)
@@ -524,7 +524,7 @@ defmodule DiscoveryApi.Data.QueryTest do
 
       assert %{
                "message" =>
-                 "Syntax Error: mismatched input 'DROP'. Expecting: ',', '.', 'AS', 'CROSS', 'EXCEPT', 'FULL', 'GROUP', 'HAVING', 'INNER', 'INTERSECT', 'JOIN', 'LEFT', 'LIMIT', 'NATURAL', 'ORDER', 'RIGHT', 'TABLESAMPLE', 'UNION', 'WHERE', <EOF>, <identifier>"
+                 "Syntax Error: Invalid X-Trino-Prepared-Statement header: line 3:3: mismatched input 'DROP'. Expecting: ',', '.', 'AS', 'CROSS', 'EXCEPT', 'FETCH', 'FOR', 'FULL', 'GROUP', 'HAVING', 'INNER', 'INTERSECT', 'JOIN', 'LEFT', 'LIMIT', 'MATCH_RECOGNIZE', 'NATURAL', 'OFFSET', 'ORDER', 'RIGHT', 'TABLESAMPLE', 'UNION', 'WHERE', 'WINDOW', <EOF>, <identifier>"
              } ==
                plain_text_post(conn, "/api/v1/query", request_body)
                |> response(400)
@@ -544,13 +544,16 @@ defmodule DiscoveryApi.Data.QueryTest do
     } do
       request_body = "SELECT * FROM #{public_table}\rSELECT * FROM #{private_table}"
 
-      assert %{
-               "message" =>
-                 "Syntax Error: mismatched input 'SELECT'. Expecting: ',', '.', 'AS', 'CROSS', 'EXCEPT', 'FULL', 'GROUP', 'HAVING', 'INNER', 'INTERSECT', 'JOIN', 'LEFT', 'LIMIT', 'NATURAL', 'ORDER', 'RIGHT', 'TABLESAMPLE', 'UNION', 'WHERE', <EOF>, <identifier>"
-             } ==
-               plain_text_post(conn, "/api/v1/query", request_body)
-               |> response(400)
-               |> Jason.decode!()
+      expected_message_content =
+        "mismatched input 'SELECT'. Expecting: ',', '.', 'AS', 'CROSS', 'EXCEPT', 'FETCH', 'FOR', 'FULL', 'GROUP', 'HAVING', 'INNER', 'INTERSECT', 'JOIN', 'LEFT', 'LIMIT', 'MATCH_RECOGNIZE', 'NATURAL', 'OFFSET', 'ORDER', 'RIGHT', 'TABLESAMPLE', 'UNION', 'WHERE', 'WINDOW', <EOF>, <identifier>"
+
+      actual_message =
+        plain_text_post(conn, "/api/v1/query", request_body)
+        |> response(400)
+        |> Jason.decode!()
+        |> Map.get("message")
+
+      assert String.contains?(actual_message, expected_message_content)
     end
 
     test "any user can't perform multiple queries separated by a semicolon", %{
@@ -563,13 +566,16 @@ defmodule DiscoveryApi.Data.QueryTest do
         SELECT * FROM #{public_table}; DROP TABLE #{private_table}
       """
 
-      assert %{
-               "message" =>
-                 "Syntax Error: mismatched input ';'. Expecting: ',', '.', 'AS', 'CROSS', 'EXCEPT', 'FULL', 'GROUP', 'HAVING', 'INNER', 'INTERSECT', 'JOIN', 'LEFT', 'LIMIT', 'NATURAL', 'ORDER', 'RIGHT', 'TABLESAMPLE', 'UNION', 'WHERE', <EOF>, <identifier>"
-             } ==
-               plain_text_post(conn, "/api/v1/query", request_body)
-               |> response(400)
-               |> Jason.decode!()
+      expected_message_content =
+        "mismatched input ';'. Expecting: ',', '.', 'AS', 'CROSS', 'EXCEPT', 'FETCH', 'FOR', 'FULL', 'GROUP', 'HAVING', 'INNER', 'INTERSECT', 'JOIN', 'LEFT', 'LIMIT', 'MATCH_RECOGNIZE', 'NATURAL', 'OFFSET', 'ORDER', 'RIGHT', 'TABLESAMPLE', 'UNION', 'WHERE', 'WINDOW', <EOF>, <identifier>"
+
+      actual_message =
+        plain_text_post(conn, "/api/v1/query", request_body)
+        |> response(400)
+        |> Jason.decode!()
+        |> Map.get("message")
+
+      assert String.contains?(actual_message, expected_message_content)
 
       assert 3 ==
                prestige_session