feat: add composite sources support

.github/workflows/ci.yml

@@ -95,6 +95,8 @@ jobs:
           sudo apt-get install postgresql-client
           psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/TileBBox.sql
           psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/table_source.sql
+          psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/points1_source.sql
+          psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/points2_source.sql
           psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/function_source.sql
         env:
           POSTGRES_HOST: localhost
@@ -126,11 +128,14 @@ jobs:
 
       - name: Test server response
         run: |
-          curl localhost:3000/public.table_source/0/0/0.pbf > table_source.pbf
-          curl localhost:3000/rpc/public.function_source/0/0/0.pbf > function_source.pbf
+          curl "localhost:3000/public.table_source/0/0/0.pbf" > table_source.pbf
+          curl "localhost:3000/public.points1,public.points2/0/0/0.pbf" > composite_source.pbf
+          curl "localhost:3000/rpc/public.function_source/0/0/0.pbf" > function_source.pbf
           ./tests/vtzero-check table_source.pbf
+          ./tests/vtzero-check composite_source.pbf
           ./tests/vtzero-check function_source.pbf
           ./tests/vtzero-show table_source.pbf
+          ./tests/vtzero-show composite_source.pbf
           ./tests/vtzero-show function_source.pbf
 
   docker:
@@ -158,7 +163,7 @@ jobs:
       - name: Build and push the Docker image
         uses: docker/build-push-action@v2
         with:
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
 

.github/workflows/grcov.yml

@@ -30,6 +30,8 @@ jobs:
           sudo apt-get install postgresql-client
           psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/TileBBox.sql
           psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/table_source.sql
+          psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/points1_source.sql
+          psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/points2_source.sql
           psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U postgres -d test -f tests/fixtures/function_source.sql
         env:
           POSTGRES_HOST: localhost

Cargo.toml

@@ -19,6 +19,7 @@ actix-rt = "1.1"
 actix-web = "3.3.2"
 docopt = "1"
 env_logger = "0.8"
+itertools = "0.10.0"
 log = "0.4"
 native-tls = "0.2"
 num_cpus = "1.13"

README.md

@@ -19,6 +19,9 @@ Martin is a [PostGIS](https://github.com/postgis/postgis) [vector tiles](https:/
   - [Table Sources List](#table-sources-list)
   - [Table Source TileJSON](#table-source-tilejson)
   - [Table Source Tiles](#table-source-tiles)
+- [Composite Sources](#composite-sources)
+  - [Composite Source TileJSON](#composite-source-tilejson)
+  - [Composite Source Tiles](#composite-source-tiles)
 - [Function Sources](#function-sources)
   - [Function Sources List](#function-sources-list)
   - [Function Source TileJSON](#function-source-tilejson)
@@ -72,17 +75,21 @@ Martin requires a database connection string. It can be passed as a command-line
 martin postgres://postgres@localhost/db
 ```
 
+Martin provides [TileJSON](https://github.com/mapbox/tilejson-spec) endpoint for each [geospatial-enabled](https://postgis.net/docs/postgis_usage.html#geometry_columns) table in your database.
+
 ## API
 
-| Method | URL                                                  | Description                                           |
-| ------ | ---------------------------------------------------- | ----------------------------------------------------- |
-| `GET`  | `/index.json`                                        | [Table Sources List](#table-sources-list)             |
-| `GET`  | `/{schema_name}.{table_name}.json`                   | [Table Source TileJSON](#table-source-tilejson)       |
-| `GET`  | `/{schema_name}.{table_name}/{z}/{x}/{y}.pbf`        | [Table Source Tiles](#table-source-tiles)             |
-| `GET`  | `/rpc/index.json`                                    | [Function Sources List](#function-sources-list)       |
-| `GET`  | `/rpc/{schema_name}.{function_name}.json`            | [Function Source TileJSON](#function-source-tilejson) |
-| `GET`  | `/rpc/{schema_name}.{function_name}/{z}/{x}/{y}.pbf` | [Function Source Tiles](#function-source-tiles)       |
-| `GET`  | `/healthz`                                           | Martin server health check: returns `200 OK`          |
+| Method | URL                                                                              | Description                                             |
+| ------ | -------------------------------------------------------------------------------- | ------------------------------------------------------- |
+| `GET`  | `/index.json`                                                                    | [Table Sources List](#table-sources-list)               |
+| `GET`  | `/{schema_name}.{table_name}.json`                                               | [Table Source TileJSON](#table-source-tilejson)         |
+| `GET`  | `/{schema_name}.{table_name}/{z}/{x}/{y}.pbf`                                    | [Table Source Tiles](#table-source-tiles)               |
+| `GET`  | `/{schema_name1}.{table_name1},...,{schema_nameN}.{table_nameN}.json`            | [Composite Source TileJSON](#composite-source-tilejson) |
+| `GET`  | `/{schema_name1}.{table_name1},...,{schema_nameN}.{table_nameN}/{z}/{x}/{y}.pbf` | [Composite Source Tiles](#composite-source-tiles)       |
+| `GET`  | `/rpc/index.json`                                                                | [Function Sources List](#function-sources-list)         |
+| `GET`  | `/rpc/{schema_name}.{function_name}.json`                                        | [Function Source TileJSON](#function-source-tilejson)   |
+| `GET`  | `/rpc/{schema_name}.{function_name}/{z}/{x}/{y}.pbf`                             | [Function Source Tiles](#function-source-tiles)         |
+| `GET`  | `/healthz`                                                                       | Martin server health check: returns `200 OK`            |
 
 ## Using with Mapbox GL JS
 
@@ -95,19 +102,48 @@ You can add a layer to the map and specify martin TileJSON endpoint as a vector
 
 ```js
 map.addLayer({
-  id: 'public.points',
-  type: 'circle',
+  id: "public.points",
+  type: "circle",
   source: {
-    type: 'vector',
-    url: 'http://localhost:3000/public.points.json',
+    type: "vector",
+    url: "http://localhost:3000/public.points.json",
   },
-  'source-layer': 'public.points',
+  "source-layer": "public.points",
   paint: {
     'circle-color': 'red',
   },
 });
 ```
 
+You can also combine multiple tables into one source with [Composite Sources](#composite-sources). Each [Table Source](#table-sources) in Composite Source can be accessed with its `{schema_name}.{table_name}` as a `source-layer` property.
+
+```js
+map.addSource("points", {
+  type: "vector",
+  url: `http://0.0.0.0:3000/public.points1,public.points2.json`,
+});
+
+map.addLayer({
+  id: "red_points",
+  type: "circle",
+  source: "points",
+  "source-layer": "public.points1",
+  paint: {
+    "circle-color": "red",
+  },
+});
+
+map.addLayer({
+  id: "blue_points",
+  type: "circle",
+  source: "points",
+  "source-layer": "public.points2",
+  paint: {
+    "circle-color": "blue",
+  },
+});
+```
+
 ## Using with Leaflet
 
 [Leaflet](https://github.com/Leaflet/Leaflet) is the leading open-source JavaScript library for mobile-friendly interactive maps.
@@ -161,6 +197,32 @@ For example, `points` table in `public` schema will be available at `/public.poi
 curl localhost:3000/public.points/0/0/0.pbf
 ```
 
+## Composite Sources
+
+Composite Sources allows combining multiple Table Sources into one. Composite Source consists of multiple Table Sources separated by comma `{schema_name1}.{table_name1},...,{schema_nameN}.{table_nameN}`
+
+Each [Table Source](#table-sources) in Composite Source can be accessed with its `{schema_name}.{table_name}` as a `source-layer` property.
+
+### Composite Source TileJSON
+
+Composite Source [TileJSON](https://github.com/mapbox/tilejson-spec) endpoint is available at `/{schema_name1}.{table_name1},...,{schema_nameN}.{table_nameN}.json`.
+
+For example, composite source for `points` and `lines` tables in `public` schema will be available at `/public.points,public.lines.json`
+
+```shell
+curl localhost:3000/public.points,public.lines.json
+```
+
+### Composite Source Tiles
+
+Composite Source tiles endpoint is available at `/{schema_name1}.{table_name1},...,{schema_nameN}.{table_nameN}/{z}/{x}/{y}.pbf`
+
+For example, composite source for `points` and `lines` tables in `public` schema will be available at `/public.points,public.lines/{z}/{x}/{y}.pbf`
+
+```shell
+curl localhost:3000/public.points,public.lines/0/0/0.pbf
+```
+
 ## Function Sources
 
 Function Source is a database function which can be used to query [vector tiles](https://github.com/mapbox/vector-tile-spec). When started, martin will look for the functions with a suitable signature. A function that takes `z integer`, `x integer`, `y integer`, and `query_params json` and returns `bytea`, can be used as a Function Source.
@@ -297,7 +359,7 @@ You can find an example of a configuration file [here](https://github.com/urbica
 
 ```yaml
 # Database connection string
-connection_string: 'postgres://postgres@localhost/db'
+connection_string: "postgres://postgres@localhost/db"
 
 # Maximum connections pool size [default: 20]
 pool_size: 20
@@ -309,7 +371,7 @@ keep_alive: 75
 worker_processes: 8
 
 # The socket address to bind [default: 0.0.0.0:3000]
-listen_addresses: '0.0.0.0:3000'
+listen_addresses: "0.0.0.0:3000"
 
 # Enable watch mode
 watch: true
@@ -410,7 +472,7 @@ docker run \
 You can use example [`docker-compose.yml`](https://raw.githubusercontent.com/urbica/martin/master/docker-compose.yml) file as a reference
 
 ```yml
-version: '3'
+version: "3"
 
 services:
   martin:

src/composite_source.rs

@@ -0,0 +1,81 @@
+use itertools::Itertools;
+use std::io;
+
+use tilejson::{TileJSON, TileJSONBuilder};
+
+use crate::db::Connection;
+use crate::source::{Query, Source, Tile, XYZ};
+use crate::table_source::TableSource;
+use crate::utils;
+
+#[derive(Clone, Debug)]
+pub struct CompositeSource {
+    pub id: String,
+    pub table_sources: Vec<TableSource>,
+}
+
+impl CompositeSource {
+    fn get_bounds_cte(&self, xyz: &XYZ) -> String {
+        let srid_bounds: String = self
+            .table_sources
+            .clone()
+            .into_iter()
+            .map(|source| source.srid)
+            .unique()
+            .map(|srid| utils::get_srid_bounds(srid, xyz))
+            .collect::<Vec<String>>()
+            .join(", ");
+
+        utils::get_bounds_cte(srid_bounds)
+    }
+
+    fn get_tile_query(&self, xyz: &XYZ) -> String {
+        let tile_query: String = self
+            .table_sources
+            .clone()
+            .into_iter()
+            .map(|source| format!("({})", source.get_tile_query(xyz)))
+            .collect::<Vec<String>>()
+            .join(" || ");
+
+        format!("SELECT {} AS tile", tile_query)
+    }
+
+    pub fn build_tile_query(&self, xyz: &XYZ) -> String {
+        let bounds_cte = self.get_bounds_cte(xyz);
+        let tile_query = self.get_tile_query(xyz);
+
+        format!("{} {}", bounds_cte, tile_query)
+    }
+}
+
+impl Source for CompositeSource {
+    fn get_id(&self) -> &str {
+        self.id.as_str()
+    }
+
+    fn get_tilejson(&self) -> Result<TileJSON, io::Error> {
+        let mut tilejson_builder = TileJSONBuilder::new();
+
+        tilejson_builder.scheme("xyz");
+        tilejson_builder.name(&self.id);
+
+        Ok(tilejson_builder.finalize())
+    }
+
+    fn get_tile(
+        &self,
+        conn: &mut Connection,
+        xyz: &XYZ,
+        _query: &Option<Query>,
+    ) -> Result<Tile, io::Error> {
+        let tile_query = self.build_tile_query(xyz);
+
+        let tile: Tile = conn
+            .query_one(tile_query.as_str(), &[])
+            .map(|row| row.get("tile"))
+            .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))?;
+
+        Ok(tile)
+    }
+}

src/dev.rs

@@ -13,9 +13,8 @@ use crate::server::AppState;
 use crate::table_source::{TableSource, TableSources};
 
 pub fn mock_table_sources() -> Option<TableSources> {
-    let id = "public.table_source";
     let source = TableSource {
-        id: id.to_owned(),
+        id: "public.table_source".to_owned(),
         schema: "public".to_owned(),
         table: "table_source".to_owned(),
         id_column: None,
@@ -28,8 +27,38 @@ pub fn mock_table_sources() -> Option<TableSources> {
         properties: HashMap::new(),
     };
 
+    let table_source1 = TableSource {
+        id: "public.points1".to_owned(),
+        schema: "public".to_owned(),
+        table: "points1".to_owned(),
+        id_column: None,
+        geometry_column: "geom".to_owned(),
+        srid: 3857,
+        extent: Some(4096),
+        buffer: Some(64),
+        clip_geom: Some(true),
+        geometry_type: None,
+        properties: HashMap::new(),
+    };
+
+    let table_source2 = TableSource {
+        id: "public.points2".to_owned(),
+        schema: "public".to_owned(),
+        table: "points2".to_owned(),
+        id_column: None,
+        geometry_column: "geom".to_owned(),
+        srid: 3857,
+        extent: Some(4096),
+        buffer: Some(64),
+        clip_geom: Some(true),
+        geometry_type: None,
+        properties: HashMap::new(),
+    };
+
     let mut table_sources: TableSources = HashMap::new();
-    table_sources.insert(id.to_owned(), Box::new(source));
+    table_sources.insert("public.table_source".to_owned(), Box::new(source));
+    table_sources.insert("public.points1".to_owned(), Box::new(table_source1));
+    table_sources.insert("public.points2".to_owned(), Box::new(table_source2));
     Some(table_sources)
 }
 

src/lib.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 extern crate log;
 
+pub mod composite_source;
 pub mod config;
 pub mod coordinator_actor;
 pub mod db;

src/scripts/get_bounds_cte.sql

@@ -0,0 +1 @@
+WITH bounds AS (SELECT {srid_bounds})

src/scripts/get_geom.sql

@@ -0,0 +1,4 @@
+SELECT
+  ST_AsMVTGeom (ST_Transform ({geometry_column}, 3857), {mercator_bounds}, {extent}, {buffer}, {clip_geom}) AS geom {properties} FROM {id}, bounds
+  WHERE
+    {geometry_column} && bounds.srid_{srid}

src/scripts/get_srid_bounds.sql

@@ -0,0 +1 @@
+ST_Transform({mercator_bounds}, {srid}) AS srid_{srid}

src/scripts/get_tile.sql

@@ -1,7 +1,2 @@
-WITH bounds AS (SELECT {mercator_bounds} as mercator, {original_bounds} as original)
-SELECT ST_AsMVT(tile, '{id}', {extent}, 'geom' {id_column}) FROM (
-  SELECT
-    ST_AsMVTGeom({geometry_column_mercator}, bounds.mercator, {extent}, {buffer}, {clip_geom}) AS geom {properties}
-  FROM {id}, bounds
-  WHERE {geometry_column} && bounds.original
-) AS tile WHERE geom IS NOT NULL
+SELECT
+  ST_AsMVT (tile, '{id}', {extent}, 'geom' {id_column}) FROM ({geom_query}) AS tile