Added support for Elasticsearch 7.6.2

Martin Spütz · Martin Spütz · commit cd0d967552e2 · 2020-05-06T13:53:31.000+02:00
diff --git a/es/index_settings.json b/es/index_settings.json
@@ -92,7 +92,7 @@
 				"type": "length"
 			},
 			"preserving_word_delimiter": { 
-				"type": "word_delimiter",
+				"type": "word_delimiter_graph",
 				"preserve_original": "true"
 			}
 		}
diff --git a/es/mappings.json b/es/mappings.json
@@ -1,9 +1,6 @@
 {
 	"place": {
 		"dynamic": "false",
-		"_all": {
-			"enabled": false
-		},
 		"_source": {
 			"excludes": [
 				"context.*"
diff --git a/es/modules/lang-painless/plugin-descriptor.properties b/es/modules/lang-painless/plugin-descriptor.properties
@@ -22,7 +22,7 @@
 description=An easy, safe and fast scripting language for Elasticsearch
 #
 # 'version': plugin's version
-version=5.5.0
+version=7.6.2
 #
 # 'name': the plugin name
 name=lang-painless
@@ -37,7 +37,7 @@ classname=org.elasticsearch.painless.PainlessPlugin
 java.version=1.8
 #
 # 'elasticsearch.version': version of elasticsearch compiled against
-elasticsearch.version=5.5.0
+elasticsearch.version=7.6.2
 ### optional elements for plugins:
 #
 # 'has.native.controller': whether or not the plugin has a native controller
diff --git a/pom.xml b/pom.xml
@@ -17,6 +17,10 @@
                     <groupId>org.elasticsearch</groupId>
                     <artifactId>jna</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-api</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency><!-- required by elasticsearch -->
@@ -38,7 +42,7 @@
             see https://github.com/elastic/elasticsearch/issues/13245 -->
             <groupId>org.elasticsearch</groupId>
             <artifactId>jna</artifactId>
-            <version>4.4.0</version>
+            <version>4.5.1</version>
         </dependency>
         <dependency>
             <groupId>postgresql</groupId>
@@ -229,7 +233,7 @@
                         <artifactItem>
                             <groupId>org.codelibs.elasticsearch.module</groupId>
                             <artifactId>lang-painless</artifactId>
-                            <version>5.5.0</version>
+                            <version>7.6.2</version>
                             <type>jar</type>
                             <overWrite>false</overWrite>
                             <outputDirectory>${project.build.directory}/es/modules/lang-painless</outputDirectory>
@@ -314,9 +318,10 @@
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <maven.compiler.encoding>UTF-8</maven.compiler.encoding>
 
-        <elasticsearch.version>5.5.0</elasticsearch.version>
-        <slf4j.version>1.7.25</slf4j.version>
-        <log4j.version>2.8.2</log4j.version>
+        <!-- <elasticsearch.version>5.5.0</elasticsearch.version> -->
+        <elasticsearch.version>7.6.2</elasticsearch.version>
+        <slf4j.version>1.7.30</slf4j.version>
+        <log4j.version>2.13.2</log4j.version>
 
         <additionalparam>-Xdoclint:none</additionalparam>
     </properties>
diff --git a/src/main/java/de/komoot/photon/JsonDumper.java b/src/main/java/de/komoot/photon/JsonDumper.java
@@ -1,6 +1,7 @@
 package de.komoot.photon;
 
 import lombok.extern.slf4j.Slf4j;
+import org.elasticsearch.common.Strings;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -25,7 +26,7 @@ public JsonDumper(String filename, String languages) throws FileNotFoundExceptio
     public void add(PhotonDoc doc) {
         try {
             writer.println("{\"index\": {}}");
-            writer.println(Utils.convert(doc, this.languages).string());
+            writer.println(Strings.toString(Utils.convert(doc, this.languages)));
         } catch (IOException e) {
             log.error("error writing json file", e);
         }
diff --git a/src/main/java/de/komoot/photon/elasticsearch/Importer.java b/src/main/java/de/komoot/photon/elasticsearch/Importer.java
@@ -7,6 +7,8 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
@@ -35,8 +37,10 @@ public Importer(Client esClient, String languages) {
     @Override
     public void add(PhotonDoc doc) {
         try {
+            XContentBuilder builder = Utils.convert(doc, languages);
+
             this.bulkRequest.add(this.esClient.prepareIndex(indexName, indexType).
-                    setSource(Utils.convert(doc, languages)).setId(doc.getUid()));
+                    setSource(builder).setId(doc.getUid()));
         } catch (IOException e) {
             log.error("could not bulk add document " + doc.getUid(), e);
             return;
@@ -65,6 +69,6 @@ public void finish() {
 
     public long count() {
         return this.esClient.search(Requests.searchRequest(indexName).types(indexType).source(SearchSourceBuilder.searchSource().size(0))).actionGet().getHits()
-                .getTotalHits();
+                .getTotalHits().value;
     }
 }
diff --git a/src/main/java/de/komoot/photon/elasticsearch/Server.java b/src/main/java/de/komoot/photon/elasticsearch/Server.java
@@ -7,8 +7,9 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.node.Node;
@@ -27,10 +28,7 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.*;
 
 /**
  * Helper class to start/stop elasticsearch node and get elasticsearch clients
@@ -54,8 +52,8 @@ public class Server {
     private Integer shards = null;
 
     protected static class MyNode extends Node {
-        public MyNode(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins) {
-            super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), classpathPlugins);
+        public MyNode(Environment environment) {
+            super(environment);
         }
     }
 
@@ -92,9 +90,9 @@ public Server start() {
                 if (index >= 0) {
                     int port = Integer.parseInt(tAddr.substring(index + 1));
                     String addrStr = tAddr.substring(0, index);
-                    trClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(addrStr, port)));
+                    trClient.addTransportAddress(new TransportAddress(new InetSocketAddress(addrStr, port)));
                 } else {
-                    trClient.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(tAddr, 9300)));
+                    trClient.addTransportAddress(new TransportAddress(new InetSocketAddress(tAddr, 9300)));
                 }
             }
 
@@ -107,9 +105,14 @@ public Server start() {
             try {
                 sBuilder.put("transport.type", "netty4").put("http.type", "netty4").put("http.enabled", "true");
                 Settings settings = sBuilder.build();
+                /*
                 Collection<Class<? extends Plugin>> lList = new LinkedList<>();
                 lList.add(Netty4Plugin.class);
-                esNode = new MyNode(settings, lList);
+                */
+
+                Environment environment = new Environment(settings, null);
+                esNode = new Node(environment);
+                //esNode = new MyNode(settings, lList);
                 esNode.start();
 
                 log.info("started elastic search node");
diff --git a/src/main/java/de/komoot/photon/query/PhotonQueryBuilder.java b/src/main/java/de/komoot/photon/query/PhotonQueryBuilder.java
@@ -5,7 +5,7 @@
 import com.vividsolutions.jts.geom.Envelope;
 import com.vividsolutions.jts.geom.Point;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
-import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery.ScoreMode;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery.ScoreMode;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.*;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
diff --git a/src/main/java/de/komoot/photon/searcher/AbstractPhotonRequestHandler.java b/src/main/java/de/komoot/photon/searcher/AbstractPhotonRequestHandler.java
@@ -26,7 +26,7 @@ public List<JSONObject> handle(R photonRequest) {
         int limit = photonRequest.getLimit();
         int extLimit = limit > 1 ? (int) Math.round(photonRequest.getLimit() * 1.5) : 1;
         SearchResponse results = elasticsearchSearcher.search(queryBuilder.buildQuery(), extLimit);
-        if (results.getHits().getTotalHits() == 0) {
+        if (results.getHits().getTotalHits().value == 0L) {
             results = elasticsearchSearcher.search(queryBuilder.withLenientMatch().buildQuery(), extLimit);
         }
         List<JSONObject> resultJsonObjects = new ConvertToJson(photonRequest.getLanguage()).convert(results);
diff --git a/src/main/java/de/komoot/photon/searcher/BaseElasticsearchSearcher.java b/src/main/java/de/komoot/photon/searcher/BaseElasticsearchSearcher.java
@@ -21,7 +21,7 @@ public BaseElasticsearchSearcher(Client client) {
     public SearchResponse search(QueryBuilder queryBuilder, Integer limit) {
         TimeValue timeout = TimeValue.timeValueSeconds(7);
         return client.prepareSearch("photon").
-                setSearchType(SearchType.QUERY_AND_FETCH).
+                setSearchType(SearchType.QUERY_THEN_FETCH).
                 setQuery(queryBuilder).
                 setSize(limit).
                 setTimeout(timeout).
diff --git a/src/main/java/de/komoot/photon/searcher/ReverseElasticsearchSearcher.java b/src/main/java/de/komoot/photon/searcher/ReverseElasticsearchSearcher.java
@@ -26,7 +26,7 @@ public SearchResponse search(QueryBuilder queryBuilder, Integer limit, Point loc
                                  Boolean locationDistanceSort) {
         TimeValue timeout = TimeValue.timeValueSeconds(7);
 
-        SearchRequestBuilder builder = client.prepareSearch("photon").setSearchType(SearchType.QUERY_AND_FETCH)
+        SearchRequestBuilder builder = client.prepareSearch("photon").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .setQuery(queryBuilder).setSize(limit).setTimeout(timeout);
 
         if (locationDistanceSort)
diff --git a/src/main/java/de/komoot/photon/utils/ConvertToJson.java b/src/main/java/de/komoot/photon/utils/ConvertToJson.java
@@ -28,10 +28,10 @@ public ConvertToJson(String lang) {
 
     @Override
     public List<JSONObject> convert(SearchResponse searchResponse) {
-        SearchHit[] hits = searchResponse.getHits().hits();
+        SearchHit[] hits = searchResponse.getHits().getHits();
         final List<JSONObject> list = Lists.newArrayListWithExpectedSize(hits.length);
         for (SearchHit hit : hits) {
-            final Map<String, Object> source = hit.getSource();
+            final Map<String, Object> source = hit.getSourceAsMap();
 
             final JSONObject feature = new JSONObject();
             feature.put(Constants.TYPE, Constants.FEATURE);
diff --git a/src/main/java/de/komoot/photon/utils/QueryToJson.java b/src/main/java/de/komoot/photon/utils/QueryToJson.java
@@ -17,8 +17,7 @@ public class QueryToJson implements OneWayConverter<QueryBuilder, String> {
     @Override
     public String convert(QueryBuilder anItem) {
         try {
-            BytesReference bytes = anItem.toXContent(JsonXContent.contentBuilder(), new ToXContent.MapParams(null))
-                    .bytes();
+            BytesReference bytes = BytesReference.bytes(anItem.toXContent(JsonXContent.contentBuilder(), new ToXContent.MapParams(null)));
 
             return bytes.utf8ToString();
         } catch (IOException e) {

Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,7 @@`
`92`	`92`	`"type": "length"`
`93`	`93`	`},`
`94`	`94`	`"preserving_word_delimiter": {`
`95`		`- "type": "word_delimiter",`
	`95`	`+ "type": "word_delimiter_graph",`
`96`	`96`	`"preserve_original": "true"`
`97`	`97`	`}`
`98`	`98`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@`
`22`	`22`	`description=An easy, safe and fast scripting language for Elasticsearch`
`23`	`23`	`#`
`24`	`24`	`# 'version': plugin's version`
`25`		`-version=5.5.0`
	`25`	`+version=7.6.2`
`26`	`26`	`#`
`27`	`27`	`# 'name': the plugin name`
`28`	`28`	`name=lang-painless`
`@@ -37,7 +37,7 @@ classname=org.elasticsearch.painless.PainlessPlugin`
`37`	`37`	`java.version=1.8`
`38`	`38`	`#`
`39`	`39`	`# 'elasticsearch.version': version of elasticsearch compiled against`
`40`		`-elasticsearch.version=5.5.0`
	`40`	`+elasticsearch.version=7.6.2`
`41`	`41`	`### optional elements for plugins:`
`42`	`42`	`#`
`43`	`43`	`# 'has.native.controller': whether or not the plugin has a native controller`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ public List<JSONObject> handle(R photonRequest) {`
`26`	`26`	`int limit = photonRequest.getLimit();`
`27`	`27`	`int extLimit = limit > 1 ? (int) Math.round(photonRequest.getLimit() * 1.5) : 1;`
`28`	`28`	`SearchResponse results = elasticsearchSearcher.search(queryBuilder.buildQuery(), extLimit);`
`29`		`- if (results.getHits().getTotalHits() == 0) {`
	`29`	`+ if (results.getHits().getTotalHits().value == 0L) {`
`30`	`30`	`results = elasticsearchSearcher.search(queryBuilder.withLenientMatch().buildQuery(), extLimit);`
`31`	`31`	`}`
`32`	`32`	`List<JSONObject> resultJsonObjects = new ConvertToJson(photonRequest.getLanguage()).convert(results);`