Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Default ESQL data partitioning to DOC #99545

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,16 @@ protected Page process(Page page) {
blockBuilders[i] = types[i].newBlockBuilder(rowsCount);
}

BytesRefBlock input = (BytesRefBlock) inputEvaluator.eval(page);
Block input = inputEvaluator.eval(page);
BytesRef spare = new BytesRef();
for (int row = 0; row < rowsCount; row++) {
if (input.isNull(row)) {
for (int i = 0; i < blockBuilders.length; i++) {
blockBuilders[i].appendNull();
for (Block.Builder blockBuilder : blockBuilders) {
blockBuilder.appendNull();
}
continue;
}
evaluator.computeRow(input, row, blockBuilders, spare);
evaluator.computeRow((BytesRefBlock) input, row, blockBuilders, spare);
}

Block[] blocks = new Block[blockBuilders.length];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ protected Page process(Page page) {
blockBuilders[i] = BytesRefBlock.newBlockBuilder(rowsCount);
}

BytesRefBlock input = (BytesRefBlock) inputEvaluator.eval(page);
Block input = inputEvaluator.eval(page);
BytesRef spare = new BytesRef();
for (int row = 0; row < rowsCount; row++) {
if (input.isNull(row)) {
Expand All @@ -72,10 +72,11 @@ protected Page process(Page page) {
continue;
}

BytesRefBlock bytesInput = (BytesRefBlock) input;
int position = input.getFirstValueIndex(row);
int valueCount = input.getValueCount(row);
if (valueCount == 1) {
Map<String, String> items = parser.apply(input.getBytesRef(position, spare).utf8ToString());
Map<String, String> items = parser.apply(bytesInput.getBytesRef(position, spare).utf8ToString());
if (items == null) {
for (int i = 0; i < fieldNames.length; i++) {
blockBuilders[i].appendNull();
Expand All @@ -91,7 +92,7 @@ protected Page process(Page page) {
String[] firstValues = new String[fieldNames.length];
boolean[] positionEntryOpen = new boolean[fieldNames.length];
for (int c = 0; c < valueCount; c++) {
Map<String, String> items = parser.apply(input.getBytesRef(position + c, spare).utf8ToString());
Map<String, String> items = parser.apply(bytesInput.getBytesRef(position + c, spare).utf8ToString());
if (items == null) {
continue;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import org.elasticsearch.client.Request;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.logging.LogManager;
import org.elasticsearch.logging.Logger;
import org.elasticsearch.test.rest.ESRestTestCase;
Expand Down Expand Up @@ -88,8 +89,11 @@ public final void test() throws Throwable {
}

protected final void doTest() throws Throwable {
RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values()));
Map<String, Object> answer = runEsql(builder.query(testCase.query).build(), testCase.expectedWarnings);
RequestObjectBuilder requestBuilder = new RequestObjectBuilder(randomFrom(XContentType.values()));
requestBuilder.query(testCase.query);
// TODO: Randomize the query pragmas
requestBuilder.pragmas(Settings.builder().put("data_partitioning", "segment").build());
Map<String, Object> answer = runEsql(requestBuilder.build(), testCase.expectedWarnings);
var expectedColumnsWithValues = loadCsvSpecValues(testCase.expectedResults);

assertNotNull(answer.get("columns"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,27 +208,27 @@ public void testColumnarMode() throws IOException {
public void testTextMode() throws IOException {
int count = randomIntBetween(0, 100);
bulkLoadTestData(count);
var builder = builder().query(fromIndex() + " | keep keyword, integer").build();
var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build();
assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null));
}

public void testCSVMode() throws IOException {
int count = randomIntBetween(0, 100);
bulkLoadTestData(count);
var builder = builder().query(fromIndex() + " | keep keyword, integer").build();
var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build();
assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|'));
}

public void testTSVMode() throws IOException {
int count = randomIntBetween(0, 100);
bulkLoadTestData(count);
var builder = builder().query(fromIndex() + " | keep keyword, integer").build();
var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build();
assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null));
}

public void testCSVNoHeaderMode() throws IOException {
bulkLoadTestData(1);
var builder = builder().query(fromIndex() + " | keep keyword, integer").build();
var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer").build();
Request request = prepareRequest();
String mediaType = attachBody(builder, request);
RequestOptions.Builder options = request.getOptions().toBuilder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public final class QueryPragmas implements Writeable {
public static final Setting<DataPartitioning> DATA_PARTITIONING = Setting.enumSetting(
DataPartitioning.class,
"data_partitioning",
DataPartitioning.SEGMENT
DataPartitioning.DOC
);

/**
Expand Down