From 604f29985c28e54df0cefbd03e0bca01954529b0 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 13 Sep 2023 09:03:41 -0700 Subject: [PATCH 1/6] Default ESQL data partitioning to DOC --- .../java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java index bb1f669dc2b43..2d8a7cecb10fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java @@ -33,7 +33,7 @@ public final class QueryPragmas implements Writeable { public static final Setting DATA_PARTITIONING = Setting.enumSetting( DataPartitioning.class, "data_partitioning", - DataPartitioning.SEGMENT + DataPartitioning.DOC ); /** From acb40542adffb5c22f5a0d96554641e92bd630bc Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 13 Sep 2023 10:30:58 -0700 Subject: [PATCH 2/6] Fix casting in Column|String ExtractOperator --- .../compute/operator/ColumnExtractOperator.java | 8 ++++---- .../compute/operator/StringExtractOperator.java | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java index 8e5244cb75226..6b13eb4e69349 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java @@ -57,16 +57,16 @@ protected Page process(Page page) { blockBuilders[i] = types[i].newBlockBuilder(rowsCount); } - BytesRefBlock input = (BytesRefBlock) inputEvaluator.eval(page); + Block input = inputEvaluator.eval(page); BytesRef spare = new BytesRef(); for (int row = 0; row < rowsCount; row++) { if (input.isNull(row)) { - for (int i = 0; i < blockBuilders.length; i++) { - blockBuilders[i].appendNull(); + for (Block.Builder blockBuilder : blockBuilders) { + blockBuilder.appendNull(); } continue; } - evaluator.computeRow(input, row, blockBuilders, spare); + evaluator.computeRow((BytesRefBlock) input, row, blockBuilders, spare); } Block[] blocks = new Block[blockBuilders.length]; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java index d512f40e0dcbb..9d4d46c5736a3 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java @@ -62,7 +62,7 @@ protected Page process(Page page) { blockBuilders[i] = BytesRefBlock.newBlockBuilder(rowsCount); } - BytesRefBlock input = (BytesRefBlock) inputEvaluator.eval(page); + Block input = inputEvaluator.eval(page); BytesRef spare = new BytesRef(); for (int row = 0; row < rowsCount; row++) { if (input.isNull(row)) { @@ -72,10 +72,11 @@ protected Page process(Page page) { continue; } + BytesRefBlock bytesInput = (BytesRefBlock) input; int position = input.getFirstValueIndex(row); int valueCount = input.getValueCount(row); if (valueCount == 1) { - Map items = parser.apply(input.getBytesRef(position, spare).utf8ToString()); + Map items = parser.apply(bytesInput.getBytesRef(position, spare).utf8ToString()); if (items == null) { for (int i = 0; i < fieldNames.length; i++) { blockBuilders[i].appendNull(); @@ -91,7 +92,7 @@ protected Page process(Page page) { String[] firstValues = new String[fieldNames.length]; boolean[] positionEntryOpen = new boolean[fieldNames.length]; for (int c = 0; c < valueCount; c++) { - Map items = parser.apply(input.getBytesRef(position + c, spare).utf8ToString()); + Map items = parser.apply(bytesInput.getBytesRef(position + c, spare).utf8ToString()); if (items == null) { continue; } From 4f8d17b2acd7a732431c816246e7477ac6495fc0 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 13 Sep 2023 11:24:19 -0700 Subject: [PATCH 3/6] Use segment in spec test --- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index e9dc848024448..8b3464fbcef9a 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -10,6 +10,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.test.rest.ESRestTestCase; @@ -88,8 +89,11 @@ public final void test() throws Throwable { } protected final void doTest() throws Throwable { - RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values())); - Map answer = runEsql(builder.query(testCase.query).build(), testCase.expectedWarnings); + RequestObjectBuilder requestBuilder = new RequestObjectBuilder(randomFrom(XContentType.values())); + requestBuilder.query(testCase.query); + // TODO: Randomize the query pragmas + requestBuilder.pragmas(Settings.builder().put("data_partitioning", "segment").build()); + Map answer = runEsql(requestBuilder.build(), testCase.expectedWarnings); var expectedColumnsWithValues = loadCsvSpecValues(testCase.expectedResults); assertNotNull(answer.get("columns")); From b05113e7f2b3c29854d5444a75b6d5fd099a400d Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 13 Sep 2023 11:42:51 -0700 Subject: [PATCH 4/6] Randomize pragmas --- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 21 +++++++++++++++++-- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 8 +++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 8b3464fbcef9a..23c29452c2a55 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -90,9 +90,26 @@ public final void test() throws Throwable { protected final void doTest() throws Throwable { RequestObjectBuilder requestBuilder = new RequestObjectBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + Settings.Builder pragmas = Settings.builder(); + if (randomBoolean()) { + pragmas.put("data_partitioning", randomFrom("doc", "segment", "shard")); + } + if (randomBoolean()) { + pragmas.put("exchange_buffer_size", between(1, 5)); + } + if (randomBoolean()) { + pragmas.put("exchange_concurrent_clients", between(1, 16)); + } + if (randomBoolean()) { + pragmas.put("task_concurrency", between(1, 16)); + } + if (randomBoolean()) { + pragmas.put("page_size", between(1, 128)); + } + requestBuilder.pragmas(pragmas.build()); + } requestBuilder.query(testCase.query); - // TODO: Randomize the query pragmas - requestBuilder.pragmas(Settings.builder().put("data_partitioning", "segment").build()); Map answer = runEsql(requestBuilder.build(), testCase.expectedWarnings); var expectedColumnsWithValues = loadCsvSpecValues(testCase.expectedResults); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 7d90cf47cae09..1adcd58307444 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -208,27 +208,27 @@ public void testColumnarMode() throws IOException { public void testTextMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null)); } public void testCSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|')); } public void testTSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null)); } public void testCSVNoHeaderMode() throws IOException { bulkLoadTestData(1); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); Request request = prepareRequest(); String mediaType = attachBody(builder, request); RequestOptions.Builder options = request.getOptions().toBuilder(); From b93c86cbe38b79e8e8311dabf350d44226ddfd9f Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 13 Sep 2023 11:45:38 -0700 Subject: [PATCH 5/6] Revert "Randomize pragmas" This reverts commit b05113e7f2b3c29854d5444a75b6d5fd099a400d. --- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 21 ++----------------- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 8 +++---- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 23c29452c2a55..8b3464fbcef9a 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -90,26 +90,9 @@ public final void test() throws Throwable { protected final void doTest() throws Throwable { RequestObjectBuilder requestBuilder = new RequestObjectBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { - Settings.Builder pragmas = Settings.builder(); - if (randomBoolean()) { - pragmas.put("data_partitioning", randomFrom("doc", "segment", "shard")); - } - if (randomBoolean()) { - pragmas.put("exchange_buffer_size", between(1, 5)); - } - if (randomBoolean()) { - pragmas.put("exchange_concurrent_clients", between(1, 16)); - } - if (randomBoolean()) { - pragmas.put("task_concurrency", between(1, 16)); - } - if (randomBoolean()) { - pragmas.put("page_size", between(1, 128)); - } - requestBuilder.pragmas(pragmas.build()); - } requestBuilder.query(testCase.query); + // TODO: Randomize the query pragmas + requestBuilder.pragmas(Settings.builder().put("data_partitioning", "segment").build()); Map answer = runEsql(requestBuilder.build(), testCase.expectedWarnings); var expectedColumnsWithValues = loadCsvSpecValues(testCase.expectedResults); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 1adcd58307444..7d90cf47cae09 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -208,27 +208,27 @@ public void testColumnarMode() throws IOException { public void testTextMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null)); } public void testCSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|')); } public void testTSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null)); } public void testCSVNoHeaderMode() throws IOException { bulkLoadTestData(1); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); Request request = prepareRequest(); String mediaType = attachBody(builder, request); RequestOptions.Builder options = request.getOptions().toBuilder(); From 9388bd04c7a389b14e4928529aeb8a3659166bd5 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 13 Sep 2023 11:52:08 -0700 Subject: [PATCH 6/6] sort --- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 7d90cf47cae09..1adcd58307444 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -208,27 +208,27 @@ public void testColumnarMode() throws IOException { public void testTextMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null)); } public void testCSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|')); } public void testTSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null)); } public void testCSVNoHeaderMode() throws IOException { bulkLoadTestData(1); - var builder = builder().query(fromIndex() + " | keep keyword, integer").build(); + var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build(); Request request = prepareRequest(); String mediaType = attachBody(builder, request); RequestOptions.Builder options = request.getOptions().toBuilder();