Skip to content

Commit

Permalink
Tiler software updates (#335)
Browse files Browse the repository at this point in the history
* Update Tiler containers

* Update config for tiler server

* Split provider layers

* Update docker file - tiler server

* Update providers

* Update ne layers -  tiler server

* Update container for tiler
  • Loading branch information
Rub21 authored Jul 5, 2024
1 parent 1719642 commit 1235354
Show file tree
Hide file tree
Showing 41 changed files with 3,535 additions and 449 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ overpass-api-db/
data/*/
!data/README.md
images/changeset-replication-job/config.yaml
values_copy.yaml
values_copy.yaml
images/tiler-server/tegola-cache/
84 changes: 38 additions & 46 deletions compose/tiler.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version: '3.8'
# version: '3.8'
services:
# ######################################################
# ### Tiler DB
Expand All @@ -8,61 +8,53 @@ services:
build:
context: ../images/tiler-db
dockerfile: Dockerfile
# ports:
# - "5433:5432"
ports:
- "5432:5432"
volumes:
- ../data/tiler-db-data:/var/lib/postgresql/data
env_file:
- ../envs/.env.tiler-db
restart: always
# ######################################################
# ### Tiler imposm for importing data
# ######################################################
tiler-imposm:
image: osmseed-tiler-imposm:v1
build:
context: ../images/tiler-imposm
dockerfile: Dockerfile
volumes:
- ../data/tiler-imposm-data:/mnt/data
# - ../images/tiler-imposm:/osm
depends_on:
- tiler-db
command:
- sh
- -c
- "sleep 60 && ./start.sh"
env_file:
- ../envs/.env.tiler-db
- ../envs/.env.tiler-imposm
healthcheck:
test: ["CMD", "./liveness.sh"]
interval: 30s
timeout: 10s
retries: 3
restart: always
# ######################################################
# ### Tiler server
# ######################################################
# # ######################################################
# # ### Tiler imposm for importing data
# # ######################################################
# tiler-imposm:
# image: osmseed-tiler-imposm:v1
# build:
# context: ../images/tiler-imposm
# dockerfile: Dockerfile
# volumes:
# - ../data/tiler-imposm-data:/mnt/data
# # - ../images/tiler-imposm:/osm
# command:
# - sh
# - -c
# - "./start.sh"
# env_file:
# - ../envs/.env.tiler-db
# - ../envs/.env.tiler-imposm
# healthcheck:
# test: ["CMD", "./liveness.sh"]
# interval: 30s
# timeout: 10s
# retries: 3
# restart: always
# # ######################################################
# # ### Tiler server
# # ######################################################
tiler-server:
image: osmseed-tiler-server:v1
# platform: linux/amd64
build:
context: ../images/tiler-server
dockerfile: Dockerfile
volumes:
- ../data/tiler-server-data:/mnt/data
- ../data/tiler-imposm-data:/mnt/data/imposm
depends_on:
- tiler-db
- tiler-imposm
- ../images/tiler-server:/app
- ../data/tiler-imposm-data:/mnt/data/imposm
ports:
- "9090:9090"
command:
- sh
- -c
- "./start.sh & ./expire-watcher.sh"
- "9090:9090"
command: "./start.sh"
env_file:
- ../envs/.env.tiler-db
- ../envs/.env.tiler-server
restart: always

- ../envs/.env.tiler-db
- ../envs/.env.tiler-server
restart: always
4 changes: 3 additions & 1 deletion envs/.env.tiler-imposm.example
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,6 @@ TILER_IMPORT_FROM=osm
TILER_IMPORT_PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf
SEQUENCE_NUMBER=4768
REPLICATION_URL=https://download.openstreetmap.fr/replication/europe/monaco/minute/
OVERWRITE_STATE=false
OVERWRITE_STATE=false
IMPORT_NATURAL_EARTH=true
IMPORT_OSM_LAND=true
12 changes: 4 additions & 8 deletions images/tiler-db/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
FROM mdillon/postgis:9.5
RUN rm /etc/apt/sources.list.d/pgdg.list
RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list
RUN sed -i '/stretch-updates/d' /etc/apt/sources.list
RUN apt-get update && apt-get -y install apt-transport-https
RUN echo "deb https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 9.5" /etc/apt/sources.list.d/pgdg.list
RUN apt-get update && apt-get install -y git ca-certificates
FROM postgis/postgis:14-3.4
RUN apt-get update
RUN apt-get install -y git ca-certificates

COPY ./config/docker-entrypoint.sh /usr/local/bin/
RUN mkdir -p /docker-entrypoint-initdb.d
COPY ./config/initdb_db.sh /docker-entrypoint-initdb.d/postgis.sh
Expand Down
24 changes: 7 additions & 17 deletions images/tiler-db/config/initdb_db.sh
Original file line number Diff line number Diff line change
@@ -1,20 +1,10 @@
#!/bin/sh
set -e

# Create the 'template_postgis' template db
"${psql[@]}" <<- 'EOSQL'
CREATE DATABASE template_postgis;
UPDATE pg_database SET datistemplate = TRUE WHERE datname = 'template_postgis';
EOSQL

# Load PostGIS into both template_database and $POSTGRES_DB
for DB in template_postgis "$POSTGRES_DB"; do
echo "Loading PostGIS extensions into $DB"
"${psql[@]}" --dbname="$DB" <<-'EOSQL'
CREATE EXTENSION IF NOT EXISTS postgis;
CREATE EXTENSION IF NOT EXISTS postgis_topology;
CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
CREATE EXTENSION IF NOT EXISTS hstore;
EOSQL
done
# Add hstore into the DB
for DB in template_postgis "$POSTGRES_DB" "${@}"; do
echo "Updating extensions '$DB'"
psql --dbname="$DB" -c "
CREATE EXTENSION IF NOT EXISTS hstore;
"
done
60 changes: 22 additions & 38 deletions images/tiler-server/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,57 +1,41 @@
FROM golang:1.14.1-alpine3.11 AS build
ENV VERSION="v0.8.1"
RUN apk update
RUN apk add musl-dev=1.1.24-r3 \
gcc \
bash \
git \
postgresql \
postgresql-contrib
FROM gospatial/tegola:v0.20.0

RUN mkdir -p /go/src/github.com/go-spatial/tegola
RUN git clone https://github.com/go-spatial/tegola.git /go/src/github.com/go-spatial/tegola
RUN cd /go/src/github.com/go-spatial/tegola && git checkout v0.13.0
RUN cd /go/src/github.com/go-spatial/tegola/cmd/tegola \
&& go build -gcflags "-N -l" -o /opt/tegola \
&& chmod a+x /opt/tegola
RUN ln -s /opt/tegola /usr/bin/tegola

RUN apk add --update \
python3 \
py-pip \
py-cffi \
py-cryptography \
&& pip install --upgrade pip \
&& apk add --virtual build-deps \
RUN apk --no-cache add \
gcc \
libffi-dev \
python3-dev \
python3 \
py3-pip \
py3-cffi \
py3-cryptography \
linux-headers \
musl-dev \
openssl-dev \
curl
bash \
git \
postgresql-client \
coreutils \
jq

# Install aws-cli and gsutil
RUN pip3 install awscli
RUN curl -sSL https://sdk.cloud.google.com | bash
RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
RUN pip install --upgrade pip && pip3 install awscli mercantile

RUN pip3 install mercantile \
&& apk del build-deps \
&& rm -rf /var/cache/apk/* \
&& apk --purge -v del py-pip

RUN apk add --update coreutils jq && rm -rf /var/cache/apk/*
RUN ln -s /opt/tegola /usr/bin/tegola

COPY ./config /opt/config/
COPY build_config.py /opt/
RUN mkdir /opt/tegola_config
RUN cd /opt/ && python build_config.py \
--template config/config.template.toml \
--providers config/providers \
--output /opt/tegola_config/config.toml

# Volumen
VOLUME /mnt/data
# Copy config and exec files
COPY ./config/config.toml /opt/tegola_config/config.toml
COPY ./tile2bounds.py .
COPY ./start.sh .
COPY ./expire-watcher.sh .
COPY ./seed-by-diffs.sh .
COPY ./tile_cache_downloader.sh .
COPY ./rm_tegola_ps.sh .

ENTRYPOINT ["/bin/bash", "-c"]
CMD ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh & ./rm_tegola_ps.sh
38 changes: 38 additions & 0 deletions images/tiler-server/build_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import os
import argparse
parser = argparse.ArgumentParser(description='Merge TOML files into a configuration file.')
parser.add_argument('--template', default='config/config.template.toml', help='Path to the configuration template file.')
parser.add_argument('--providers', default='config/providers', help='Directory containing provider TOML files.')
parser.add_argument('--output', default='config/config.toml', help='Output configuration file path.')
args = parser.parse_args()

config_template_file = args.template
providers_dir = args.providers
output_file_path = args.output
toml_files = [file for file in os.listdir(providers_dir) if file.endswith(".toml")]

# Read TOML files
new_configs = {}
for toml_file in toml_files:
dir_toml_file = os.path.join(providers_dir, toml_file)
with open(dir_toml_file, "r") as file:
new_configs[dir_toml_file] = file.read()

with open(config_template_file, "r") as main_file:
content = main_file.read()

# Replace the content of main.toml with the content read from other TOML files
for toml_file, toml_file_content in new_configs.items():
print(f"Copy {toml_file} to config.toml")
section_header = "[['{}']]".format(toml_file.replace("config/", ""))
indentation_level = content.find(section_header)
if indentation_level != -1:
# Find the appropriate number of tabs or spaces for indentation
preceding_newline = content.rfind('\n', 0, indentation_level)
indentation = content[preceding_newline + 1:indentation_level]
toml_file_content = f"###### From {toml_file} \n" + toml_file_content
new_values=toml_file_content.replace("\n", "\n" + indentation)
content = content.replace(section_header, new_values)

with open(output_file_path, "w") as output_file:
output_file.write(content)
Loading

0 comments on commit 1235354

Please sign in to comment.