Skip to content

Commit d27cf8c

Browse files
tebbentebben
committed
Added datasets address and zipcode
1 parent 29d0402 commit d27cf8c

File tree

8 files changed

+107
-4
lines changed

8 files changed

+107
-4
lines changed

‎Dockerfile.load

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ COPY ./data/download/geocodeur_segment.parquet /data/geocodeur_segment.parquet
3434
COPY ./data/download/geocodeur_water.parquet /data/geocodeur_water.parquet
3535
COPY ./data/download/geocodeur_poi.parquet /data/geocodeur_poi.parquet
3636
COPY ./data/download/geocodeur_infra.parquet /data/geocodeur_infra.parquet
37+
COPY ./data/download/geocodeur_address.parquet /data/geocodeur_address.parquet
38+
COPY ./data/download/geocodeur_zipcode.parquet /data/geocodeur_zipcode.parquet
3739

3840
# Set the environment variables
3941
ENV GEOCODEUR_CONFIG_PATH="/config/geocodeur.conf"

‎scripts/download.sh

+3
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,6 @@ overturemaps download --bbox "$bbox" -t water -f geoparquet -o "$download_direct
3232

3333
echo "Downloading infrastructure data"
3434
overturemaps download --bbox "$bbox" -t infrastructure -f geoparquet -o "$download_directory/infrastructure.geoparquet"
35+
36+
echo "Downloading address data"
37+
overturemaps download --bbox "$bbox" -t address -f geoparquet -o "$download_directory/address.geoparquet"

‎src/api/handlers/geocode.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ import (
1515

1616
type GeocodeInput struct {
1717
Query string `required:"true" json:"q" query:"q" doc:"The search term to find a feature, the geocoder handles incomplete names and falls back to fuzzy search for typing errors. This way things as 'kerkstr ams' and 'kerkst masterdam' can still be found" example:"President Kennedylaan Amsterdam"`
18-
Limit uint16 `required:"falase" json:"limit" query:"limit" doc:"Maximum number of results to return" minimum:"1" maximum:"100" default:"10"`
19-
Class []string `required:"false" json:"class" query:"class" doc:"Filter results by class, this is a comma separated list. Leave empty to query on all classes" enum:"division,water,road,poi,infra" default:"division,water,road,infra,poi" example:"division,water,road,poi,infra" uniqueItems:"true"`
18+
Limit uint16 `required:"false" json:"limit" query:"limit" doc:"Maximum number of results to return" minimum:"1" maximum:"100" default:"10"`
19+
Class []string `required:"false" json:"class" query:"class" doc:"Filter results by class, this is a comma separated list. Leave empty to query on all classes" enum:"division,water,road,address,zipcode,poi,infra" default:"division,water,road,address,zipcode,infra,poi" example:"division,water,road,poi,infra" uniqueItems:"true"`
2020
Geom bool `required:"false" json:"geom" query:"geom" doc:"Include the geometry of the feature in the result" default:"false"`
2121
}
2222

‎src/database/setup.go

+2
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ func CreateDB(config settings.Config) {
6969
processParquet(pool, fmt.Sprintf("%s%s", config.Process.Folder, "geocodeur_water.parquet"))
7070
processParquet(pool, fmt.Sprintf("%s%s", config.Process.Folder, "geocodeur_poi.parquet"))
7171
processParquet(pool, fmt.Sprintf("%s%s", config.Process.Folder, "geocodeur_infra.parquet"))
72+
processParquet(pool, fmt.Sprintf("%s%s", config.Process.Folder, "geocodeur_address.parquet"))
73+
processParquet(pool, fmt.Sprintf("%s%s", config.Process.Folder, "geocodeur_zipcode.parquet"))
7274

7375
log.Info("Creating foreign key overture_search -> overture")
7476
err = createForeignKey(pool)

‎src/preprocess/preprocess.go

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ func ProcessAll() {
1616
process("water", queries.WaterQuery)
1717
process("poi", queries.PoiQuery)
1818
process("infra", queries.InfraQuery)
19+
process("address", queries.AddressQuery)
20+
process("zipcode", queries.ZipcodeQuery)
1921
}
2022

2123
func process(name string, query string) {

‎src/preprocess/queries/address_query.go

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
describe select * from read_parquet('address.geoparquet');
3+
┌────────────────┬────────────────────────────────────────────────────────────────────────────────────────┬─────────┬─────────┬─────────┬─────────┐
4+
│ column_name │ column_type │ null │ key │ default │ extra │
5+
│ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │
6+
├────────────────┼────────────────────────────────────────────────────────────────────────────────────────┼─────────┼─────────┼─────────┼─────────┤
7+
│ id │ VARCHAR │ YES │ │ │ │
8+
│ geometry │ BLOB │ YES │ │ │ │
9+
│ bbox │ STRUCT(xmin FLOAT, xmax FLOAT, ymin FLOAT, ymax FLOAT) │ YES │ │ │ │
10+
│ country │ VARCHAR │ YES │ │ │ │
11+
│ postcode │ VARCHAR │ YES │ │ │ │
12+
│ street │ VARCHAR │ YES │ │ │ │
13+
│ number │ VARCHAR │ YES │ │ │ │
14+
│ unit │ VARCHAR │ YES │ │ │ │
15+
│ address_levels │ STRUCT("value" VARCHAR)[] │ YES │ │ │ │
16+
│ version │ INTEGER │ YES │ │ │ │
17+
│ sources │ STRUCT(property VARCHAR, dataset VARCHAR, record_id VARCHAR, update_time VARCHAR, co… │ YES │ │ │ │
18+
├────────────────┴────────────────────────────────────────────────────────────────────────────────────────┴─────────┴─────────┴─────────┴─────────┤
19+
│ 11 rows 6 columns │
20+
└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
21+
*/
22+
23+
package queries
24+
25+
var AddressQuery = `
26+
INSTALL spatial;
27+
LOAD spatial;
28+
29+
COPY (
30+
WITH clip AS (
31+
SELECT
32+
CASE
33+
WHEN '%COUNTRY%' != '' THEN (SELECT geometry from read_parquet('%DATADIR%division_area.geoparquet') where lower(names.primary) = '%COUNTRY%' and subtype = 'country')
34+
ELSE ST_GeomFromText('POLYGON ((-180 -90, 180 -90, 180 90, -180 90, -180 -90))')
35+
END AS geom
36+
)
37+
SELECT
38+
a.id,
39+
a.street || ' ' || a.number AS name,
40+
ST_AsText(a.geometry) AS geom,
41+
'address' as class,
42+
'address' as subclass,
43+
array_to_string([x.value for x in address_levels], ';') as relation
44+
FROM
45+
read_parquet('%DATADIR%address.geoparquet') AS a, clip AS b
46+
WHERE
47+
ST_Within(a.geometry, b.geom)
48+
) TO '%DATADIR%geocodeur_address.parquet' (FORMAT 'PARQUET');
49+
`

‎src/preprocess/queries/zipcode_query.go

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package queries
2+
3+
var ZipcodeQuery = `
4+
INSTALL spatial;
5+
LOAD spatial;
6+
7+
COPY (
8+
WITH clip AS (
9+
SELECT
10+
CASE
11+
WHEN '%COUNTRY%' != '' THEN (SELECT geometry from read_parquet('%DATADIR%division_area.geoparquet') where lower(names.primary) = '%COUNTRY%' and subtype = 'country')
12+
ELSE ST_GeomFromText('POLYGON ((-180 -90, 180 -90, 180 90, -180 90, -180 -90))')
13+
END AS geom
14+
),
15+
zips AS (
16+
SELECT
17+
ANY_VALUE(id) as id,
18+
postcode as zipcode,
19+
ST_ConvexHull(ST_Union_Agg(geometry)) AS geom
20+
FROM
21+
read_parquet('%DATADIR%address.geoparquet')
22+
GROUP BY
23+
postcode
24+
)
25+
SELECT
26+
a.id,
27+
a.zipcode AS name,
28+
ST_AsText(a.geom) AS geom,
29+
'zipcode' as class,
30+
'zipcode' as subclass,
31+
NULL::VARCHAR as relation
32+
FROM
33+
zips AS a, clip AS b
34+
WHERE
35+
ST_Within(a.geom, b.geom)
36+
) TO '%DATADIR%geocodeur_zipcode.parquet' (FORMAT 'PARQUET');
37+
`

‎src/service/geocode.go

+10-2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ const (
3434
Water Class = "water"
3535
Poi Class = "poi"
3636
Infra Class = "infra"
37+
Address Class = "address"
38+
Zipcode Class = "zipcode"
3739
)
3840

3941
func StringToClass(s string) (Class, error) {
@@ -48,6 +50,10 @@ func StringToClass(s string) (Class, error) {
4850
return Poi, nil
4951
case string(Infra):
5052
return Infra, nil
53+
case string(Address):
54+
return Address, nil
55+
case string(Zipcode):
56+
return Zipcode, nil
5157
default:
5258
return "", fmt.Errorf("class %s not found", s)
5359
}
@@ -63,7 +69,7 @@ type GeocodeOptions struct {
6369
func (g GeocodeOptions) ClassesToSqlArray() string {
6470
classes := g.Classes
6571
if classes == nil || len(classes) == 0 {
66-
classes = []Class{Division, Road, Water, Poi, Infra}
72+
classes = []Class{Division, Road, Water, Poi, Infra, Address, Zipcode}
6773
}
6874

6975
lowerClasses := make([]string, len(classes))
@@ -190,7 +196,9 @@ func createGeocodeQuery(options GeocodeOptions, input string) string {
190196
WHEN a.class = 'water' THEN 2
191197
WHEN a.class = 'road' THEN 3
192198
WHEN a.class = 'infra' THEN 4
193-
WHEN a.class = 'poi' THEN 5
199+
WHEN a.class = 'address' THEN 5
200+
WHEN a.class = 'zipcode' THEN 6
201+
WHEN a.class = 'poi' THEN 7
194202
ELSE 100
195203
END AS class_score,
196204
CASE

0 commit comments

Comments
 (0)