Merge pull request #799 from lonvia/port-to-opensearch

Add an experimental OpenSearch-based version

Author: lonvia

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
           cache: 'gradle'
 
       - name: Compile project
-        run: ./gradlew assemble --no-daemon
+        run: ./gradlew app:es_embedded:assemble --no-daemon
 
       - uses: actions/checkout@v4
         with:
@@ -114,8 +114,9 @@ jobs:
 
       - name: Import Photon
         run: |
-            java -jar build/libs/photon-*.jar -nominatim-import -database nominatim -user runner -password foobar
-            java -jar build/libs/photon-*.jar -nominatim-update-init-for runner -database nominatim -user runner -password foobar
+          PHOTON_VERSION=`grep 'version =' buildSrc/shared.gradle | head -n 1 | sed "s:.*= '::;s:'.*::"`
+          java -jar target/photon-${PHOTON_VERSION}.jar -nominatim-import -database nominatim -user runner -password foobar
+          java -jar target/photon-${PHOTON_VERSION}.jar -nominatim-update-init-for runner -database nominatim -user runner -password foobar
 
       - name: Update Nominatim
         run: |
@@ -128,4 +129,5 @@ jobs:
 
       - name: Update Photon
         run: |
-          java -jar build/libs/photon-*.jar -nominatim-update -database nominatim -user runner -password foobar
+          PHOTON_VERSION=`grep 'version =' buildSrc/shared.gradle | head -n 1 | sed "s:.*= '::;s:'.*::"`
+          java -jar target/photon-${PHOTON_VERSION}.jar -nominatim-update -database nominatim -user runner -password foobar

.gitignore

@@ -1,6 +1,8 @@
 target/
 photon_data/
 dependency-reduced-pom.xml
+.gradle/
+app/*/build
 
 .idea/
 *.iml

README.md

@@ -52,10 +52,26 @@ photon uses [gradle](https://gradle.org) for building. To build the package
 from source make sure you have a JDK installed. Then run:
 
 ```
-./gradlew build
+./gradlew app:es_embedded:build
 ```
 
-This will build and test photon. The final jar cn be found in `build/libs`.
+This will build and test photon. The final jar can be found in `target`.
+
+#### Experimental OpenSearch version
+
+The repository also contains a version that runs against the latest
+version of [OpenSearch](https://opensearch.org/). This version is still
+experimental. To build the OpenSearch version run:
+
+```
+./gradlew app:opensearch:build
+```
+
+The final jar can be found in `target/photon-opensearch-<VERSION>.jar`.
+
+Indexes produced by this version are not compatible with the ElasticSearch
+version. There are no prebuilt indexes available. You need to create your
+own export from a Nominatim database. See 'Customized Search Data' below.
 
 ### Usage
 
@@ -71,7 +87,7 @@ Check the URL `http://localhost:2322/api?q=berlin` to see if photon is running w
 
 To enable CORS (cross-site requests), use `-cors-any` to allow any origin or `-cors-origin` with a specific origin as the argument. By default, CORS support is disabled.
 
-Discover more of photon's featurse with its usage `java -jar photon-*.jar -h`. The available options are as follows:
+Discover more of photon's features with its usage `java -jar photon-*.jar -h`. The available options are as follows:
 
 ```
 -h                    Show help / usage

build.gradle app/es_embedded/build.gradle

@@ -3,85 +3,42 @@ plugins {
   id 'application'
 }
 
-group = 'de.komoot.photon'
-version = '0.5.0'
+apply from: rootProject.file('buildSrc/shared.gradle')
 
-description = "Geocoder for OSM data"
-
-distZip.enabled = false
-distTar.enabled = false
-shadowDistZip.enabled = false
-shadowDistTar.enabled = false
-
-application {
-    mainClass = 'de.komoot.photon.App';
-}
-
-java {
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
-}
-
-repositories {
-     maven { url "https://www.datanucleus.org/downloads/maven2/" }
-     mavenCentral()
-}
-
-configurations {
-    runtimePlugins.extendsFrom runtimeOnly {
-        canBeResolved = true
-    }
-}
+description = "Geocoder for OSM data (ElasticSearch-based version)"
 
 sourceSets {
     main {
         resources {
-            srcDir 'build/es'
             srcDir 'es'
+            srcDir 'build/es'
+        }
+    }
+    test {
+        java {
+            srcDir 'src/test/java'
         }
     }
 }
 
-dependencies {
-    implementation('org.elasticsearch:elasticsearch:5.6.16') {
-        exclude(module: 'log4j-api')
+configurations {
+    runtimePlugins.extendsFrom runtimeOnly {
+        canBeResolved = true
     }
+}
+
+dependencies {
+    implementation 'org.elasticsearch:elasticsearch:5.6.16'
     implementation 'org.elasticsearch.plugin:transport-netty4-client:5.6.16'
-    implementation 'org.apache.logging.log4j:log4j-core:2.23.1'
-    implementation 'org.apache.logging.log4j:log4j-api:2.23.1'
     implementation('org.elasticsearch.client:transport:5.6.16') {
         exclude(module: 'commons-logging')
     }
-    implementation 'org.postgresql:postgresql:42.7.2'
-    implementation 'org.slf4j:slf4j-api:2.0.13'
-    implementation 'org.apache.logging.log4j:log4j-slf4j2-impl:2.23.1'
-    implementation 'com.beust:jcommander:1.82'
-    implementation 'org.apache.commons:commons-lang3:3.14.0'
-    implementation 'org.springframework:spring-jdbc:5.3.32'
-    implementation('org.apache.commons:commons-dbcp2:2.12.0') {
-exclude(module: 'commons-logging')
-    }
-    implementation 'org.locationtech.jts:jts-core:1.19.0'
-    implementation 'com.sparkjava:spark-core:2.9.4'
-    implementation 'net.postgis:postgis-jdbc:2023.1.0'
-    implementation 'org.json:json:20240303'
-
-    testImplementation(platform("org.junit:junit-bom:5.10.2"))
-    testImplementation 'com.h2database:h2:2.2.224'
-    testImplementation 'org.junit.jupiter:junit-jupiter'
-    testImplementation 'org.mockito:mockito-core:5.11.0'
-
-    testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
 
     runtimePlugins 'org.codelibs.elasticsearch.module:lang-painless:5.6.16'
     runtimePlugins 'org.ow2.asm:asm-debug-all:5.1'
     runtimePlugins 'org.antlr:antlr4-runtime:4.5.1-1'
 }
 
-tasks.named('test') {
-    useJUnitPlatform()
-}
-
 task copyLibs(type: Copy){
     from configurations.runtimePlugins {
         into layout.buildDirectory.dir('es/modules/lang-painless')
@@ -100,6 +57,8 @@ tasks.named('processResources') {
 
 shadowJar {
     mergeServiceFiles()
+    destinationDirectory.set(rootProject.file('target'))
+    archiveBaseName.set('photon')
     archiveClassifier.set('')
 
     exclude '**/module-info.class'

es/__init__.py app/es_embedded/es/__init__.py

@@ -1,5 +1,6 @@
 package de.komoot.photon;
 
+import de.komoot.photon.elasticsearch.PhotonDocConverter;
 import org.slf4j.Logger;
 
 import java.io.FileNotFoundException;
@@ -26,7 +27,7 @@ public JsonDumper(String filename, String[] languages, String[] extraTags) throw
     public void add(PhotonDoc doc, int objectId) {
         try {
             writer.println("{\"index\": {}}");
-            writer.println(Utils.convert(doc, languages, extraTags).string());
+            writer.println(PhotonDocConverter.convert(doc, languages, extraTags).string());
         } catch (IOException e) {
             LOGGER.error("Error writing json file", e);
         }

es/index_settings.json app/es_embedded/es/index_settings.json

@@ -1,10 +1,11 @@
-package de.komoot.photon.elasticsearch;
+package de.komoot.photon;
 
 import de.komoot.photon.DatabaseProperties;
 import de.komoot.photon.Importer;
 import de.komoot.photon.Updater;
 import de.komoot.photon.searcher.ReverseHandler;
 import de.komoot.photon.searcher.SearchHandler;
+import de.komoot.photon.elasticsearch.*;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
@@ -40,16 +41,6 @@
 public class Server {
     private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);
 
-    /**
-     * Database version created by new imports with the current code.
-     *
-     * Format must be: major.minor.patch-dev
-     *
-     * Increase to next to be released version when the database layout
-     * changes in an incompatible way. If it is already at the next released
-     * version, increase the dev version.
-     */
-    private static final String DATABASE_VERSION = "0.3.6-1";
     public static final String PROPERTY_DOCUMENT_ID = "DATABASE_PROPERTIES";
 
     private static final String BASE_FIELD = "document_properties";
@@ -231,7 +222,7 @@ private void deleteIndex() {
      */
     public void saveToDatabase(DatabaseProperties dbProperties) throws IOException  {
         final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(BASE_FIELD)
-                        .field(FIELD_VERSION, DATABASE_VERSION)
+                        .field(FIELD_VERSION, DatabaseProperties.DATABASE_VERSION)
                         .field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages()))
                         .field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null)
                         .endObject().endObject();
@@ -264,8 +255,8 @@ public void loadFromDatabase(DatabaseProperties dbProperties) {
         }
 
         String version = properties.getOrDefault(FIELD_VERSION, "");
-        if (!DATABASE_VERSION.equals(version)) {
-            LOGGER.error("Database has incompatible version '{}'. Expected: {}", version, DATABASE_VERSION);
+        if (!DatabaseProperties.DATABASE_VERSION.equals(version)) {
+            LOGGER.error("Database has incompatible version '{}'. Expected: {}", version, DatabaseProperties.DATABASE_VERSION);
             throw new RuntimeException("Incompatible database.");
         }
 

es/mappings.json app/es_embedded/es/mappings.json

@@ -34,7 +34,7 @@ public void add(PhotonDoc doc, int objectId) {
         String uid = doc.getUid(objectId);
         try {
             this.bulkRequest.add(this.esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
-                    setSource(Utils.convert(doc, languages, extraTags)).setId(uid));
+                    setSource(PhotonDocConverter.convert(doc, languages, extraTags)).setId(uid));
         } catch (IOException e) {
             LOGGER.error("Could not bulk add document {}", uid, e);
             return;