From 8dd927efe563a90c5065046c333e8f7f4aa12234 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 01:06:29 -0500 Subject: [PATCH 001/104] first attempt to upgrade express to v5 Co-Authored-By: Andrew Calcutt --- package-lock.json | 552 +++++++++++++++++++++++++++++------------- package.json | 2 +- src/serve_data.js | 246 +++++++++---------- src/serve_font.js | 48 ++-- src/serve_rendered.js | 499 ++++++++++++++++++-------------------- src/serve_style.js | 37 ++- src/server.js | 89 +++---- test/setup.js | 4 +- 8 files changed, 827 insertions(+), 650 deletions(-) diff --git a/package-lock.json b/package-lock.json index 408b6523e..d8d6a3dd3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "color": "4.2.3", "commander": "12.1.0", "cors": "2.8.5", - "express": "4.19.2", + "express": "5.0.1", "handlebars": "4.7.8", "http-shutdown": "1.2.2", "morgan": "1.10.0", @@ -1721,17 +1721,44 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.0.tgz", + "integrity": "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==", + "dependencies": { + "mime-db": "^1.53.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", @@ -1924,9 +1951,9 @@ } }, "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz", + "integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA==" }, "node_modules/array-ify": { "version": "1.0.0", @@ -2041,41 +2068,58 @@ } }, "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.0.2.tgz", + "integrity": "sha512-SNMk0OONlQ01uk8EPeiBvTW7W4ovpL5b1O3t1sjpPgfxOQ6BqQJ6XjxinDPR79Z6HdcD5zBBwr5ssiTlgdNztQ==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", + "debug": "3.1.0", "destroy": "1.2.0", "http-errors": "2.0.0", - "iconv-lite": "0.4.24", + "iconv-lite": "0.5.2", "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" + "qs": "6.13.0", + "raw-body": "^3.0.0", + "type-is": "~1.6.18" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": ">=18" } }, "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "dependencies": { "ms": "2.0.0" } }, + "node_modules/body-parser/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/body-parser/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/body-parser/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -2156,12 +2200,19 @@ } }, "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "license": "MIT", "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -2492,9 +2543,9 @@ "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==" }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", + "integrity": "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==", "dependencies": { "safe-buffer": "5.2.1" }, @@ -2565,17 +2616,21 @@ } }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/cookiejar": { "version": "2.1.4", @@ -2742,6 +2797,23 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/define-properties": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", @@ -2867,9 +2939,9 @@ "dev": true }, "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "engines": { "node": ">= 0.8" } @@ -2976,6 +3048,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz", @@ -3445,58 +3538,65 @@ } }, "node_modules/express": { - "version": "4.19.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", - "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.2", - "content-disposition": "0.5.4", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.0.1.tgz", + "integrity": "sha512-ORF7g6qGnD+YtUG9yx4DFoqCShNMmUKiXuT5oWMHiOvt/4WFbHC6yCwQMTSBMno7AqntNCAzzcnnjowRkTL9eQ==", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.0.1", + "content-disposition": "^1.0.0", "content-type": "~1.0.4", - "cookie": "0.6.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", + "cookie": "0.7.1", + "cookie-signature": "^1.2.1", + "debug": "4.3.6", "depd": "2.0.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", - "fresh": "0.5.2", + "finalhandler": "^2.0.0", + "fresh": "2.0.0", "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", + "merge-descriptors": "^2.0.0", "methods": "~1.1.2", + "mime-types": "^3.0.0", "on-finished": "2.4.1", + "once": "1.4.0", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", + "router": "^2.0.0", "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", + "send": "^1.1.0", + "serve-static": "^2.1.0", "setprototypeof": "1.2.0", "statuses": "2.0.1", - "type-is": "~1.6.18", + "type-is": "^2.0.0", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "engines": { - "node": ">= 0.10.0" + "node": ">= 18" } }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" + "node_modules/express/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "engines": { + "node": ">= 0.6" } }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "node_modules/express/node_modules/mime-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.0.tgz", + "integrity": "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==", + "dependencies": { + "mime-db": "^1.53.0" + }, + "engines": { + "node": ">= 0.6" + } }, "node_modules/extend-shallow": { "version": "2.0.1", @@ -3599,9 +3699,9 @@ } }, "node_modules/finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.0.0.tgz", + "integrity": "sha512-MX6Zo2adDViYh+GcxxB1dpO43eypOGUOL12rLCOTMQv/DfIbpSJUy4oQIIZhVZkH9e+bZWKMon0XHFEju16tkQ==", "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", @@ -3623,6 +3723,14 @@ "ms": "2.0.0" } }, + "node_modules/finalhandler/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/finalhandler/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -3735,11 +3843,11 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/fs-minipass": { @@ -3772,9 +3880,13 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/function.prototype.name": { "version": "1.1.5", @@ -3877,13 +3989,19 @@ } }, "node_modules/get-intrinsic": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", - "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "license": "MIT", "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4146,11 +4264,12 @@ } }, "node_modules/has-property-descriptors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", - "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.1.1" + "es-define-property": "^1.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4197,6 +4316,18 @@ "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -4295,9 +4426,9 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz", + "integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -4632,6 +4763,11 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==" + }, "node_modules/is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -5305,11 +5441,11 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/memorystream": { @@ -5382,9 +5518,15 @@ } }, "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/merge-stream": { "version": "2.0.0", @@ -5427,17 +5569,6 @@ "node": ">=8.6" } }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -5908,6 +6039,7 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "optional": true, "engines": { "node": ">= 0.6" } @@ -6318,9 +6450,13 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", + "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6543,9 +6679,12 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", + "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "engines": { + "node": ">=16" + } }, "node_modules/path-type": { "version": "4.0.0", @@ -6712,11 +6851,12 @@ } }, "node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -6776,19 +6916,30 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz", + "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", - "iconv-lite": "0.4.24", + "iconv-lite": "0.6.3", "unpipe": "1.0.0" }, "engines": { "node": ">= 0.8" } }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", @@ -7086,6 +7237,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/router": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.0.0.tgz", + "integrity": "sha512-dIM5zVoG8xhC6rnSN8uoAgFARwTE7BQs8YwHEvK0VCmfxQXMaOuA1uiR1IPwsW7JyK5iTt7Od/TC9StasS2NPQ==", + "dependencies": { + "array-flatten": "3.0.0", + "is-promise": "4.0.0", + "methods": "~1.1.2", + "parseurl": "~1.3.3", + "path-to-regexp": "^8.0.0", + "setprototypeof": "1.2.0", + "utils-merge": "1.0.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -7186,41 +7354,35 @@ } }, "node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.1.0.tgz", + "integrity": "sha512-v67WcEouB5GxbTWL/4NeToqcZiAWEq90N888fczVArY8A79J0L4FD7vj5hm3eUMua5EpoQ59wa/oovY6TLvRUA==", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "debug": "^4.3.5", + "destroy": "^1.2.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^0.5.2", + "http-errors": "^2.0.0", + "mime-types": "^2.1.35", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" } }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" + "node_modules/send/node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" } }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, "node_modules/send/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -7236,17 +7398,17 @@ } }, "node_modules/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.1.0.tgz", + "integrity": "sha512-A3We5UfEjG8Z7VkDv6uItWw6HY2bBSBJT1KtVESn6EOoOr2jAxNhxWCLY3jDE2WcuHXByWju74ck3ZgLwL8xmA==", "dependencies": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.18.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.0.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" } }, "node_modules/set-blocking": { @@ -7254,6 +7416,23 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/set-value": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", @@ -7406,13 +7585,18 @@ "dev": true }, "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "license": "MIT", "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -8110,12 +8294,32 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.0.tgz", + "integrity": "sha512-gd0sGezQYCbWSbkZr75mln4YBidWUN60+devscpLF5mtRDUpiaTvKpBNrdaCvel1NdR2k6vclXybU5fBd2i+nw==", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.0.tgz", + "integrity": "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==", + "dependencies": { + "mime-db": "^1.53.0" }, "engines": { "node": ">= 0.6" diff --git a/package.json b/package.json index ea5e2fb1c..e6a609bbe 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "color": "4.2.3", "commander": "12.1.0", "cors": "2.8.5", - "express": "4.19.2", + "express": "5.0.1", "handlebars": "4.7.8", "http-shutdown": "1.2.2", "morgan": "1.10.0", diff --git a/src/serve_data.js b/src/serve_data.js index 995b6564a..d9bea6871 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -21,148 +21,144 @@ export const serve_data = { init: (options, repo) => { const app = express().disable('x-powered-by'); - app.get( - '/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)', - async (req, res, next) => { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const tileJSONFormat = item.tileJSON.format; - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - let format = req.params.format; - if (format === options.pbfAlias) { - format = 'pbf'; - } - if ( - format !== tileJSONFormat && - !(format === 'geojson' && tileJSONFormat === 'pbf') - ) { - return res.status(404).send('Invalid format'); - } - if ( - z < item.tileJSON.minzoom || - 0 || - x < 0 || - y < 0 || - z > item.tileJSON.maxzoom || - x >= Math.pow(2, z) || - y >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } - if (item.sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(item.source, z, x, y); - if (tileinfo == undefined || tileinfo.data == undefined) { - return res.status(404).send('Not found'); - } else { - let data = tileinfo.data; - let headers = tileinfo.header; - if (tileJSONFormat === 'pbf') { - if (options.dataDecoratorFunc) { - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } + app.get('/:id/:z/:x/:y.:format', async (req, res) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + const tileJSONFormat = item.tileJSON.format; + const z = req.params.z | 0; + const x = req.params.x | 0; + const y = req.params.y | 0; + let format = req.params.format; + if (format === options.pbfAlias) { + format = 'pbf'; + } + if ( + format !== tileJSONFormat && + !(format === 'geojson' && tileJSONFormat === 'pbf') + ) { + return res.status(404).send('Invalid format'); + } + if ( + z < item.tileJSON.minzoom || + 0 || + x < 0 || + y < 0 || + z > item.tileJSON.maxzoom || + x >= Math.pow(2, z) || + y >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } + if (item.sourceType === 'pmtiles') { + let tileinfo = await getPMtilesTile(item.source, z, x, y); + if (tileinfo == undefined || tileinfo.data == undefined) { + return res.status(404).send('Not found'); + } else { + let data = tileinfo.data; + let headers = tileinfo.header; + if (tileJSONFormat === 'pbf') { + if (options.dataDecoratorFunc) { + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } + } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); } - data = JSON.stringify(geojson); } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); + data = JSON.stringify(geojson); + } + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); - data = await gzipP(data); + data = await gzipP(data); - return res.status(200).send(data); - } - } else if (item.sourceType === 'mbtiles') { - item.source.getTile(z, x, y, async (err, data, headers) => { - let isGzipped; - if (err) { - if (/does not exist/.test(err.message)) { - return res.status(204).send(); - } else { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } + return res.status(200).send(data); + } + } else if (item.sourceType === 'mbtiles') { + item.source.getTile(z, x, y, async (err, data, headers) => { + let isGzipped; + if (err) { + if (/does not exist/.test(err.message)) { + return res.status(204).send(); } else { - if (data == null) { - return res.status(404).send('Not found'); - } else { - if (tileJSONFormat === 'pbf') { - isGzipped = - data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - if (options.dataDecoratorFunc) { - if (isGzipped) { - data = await gunzipP(data); - isGzipped = false; - } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - + return res + .status(500) + .header('Content-Type', 'text/plain') + .send(err.message); + } + } else { + if (data == null) { + return res.status(404).send('Not found'); + } else { + if (tileJSONFormat === 'pbf') { + isGzipped = + data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; + if (options.dataDecoratorFunc) { if (isGzipped) { data = await gunzipP(data); isGzipped = false; } + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + } + } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } - } - data = JSON.stringify(geojson); + if (isGzipped) { + data = await gunzipP(data); + isGzipped = false; } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); - if (!isGzipped) { - data = await gzipP(data); + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); + } } + data = JSON.stringify(geojson); + } + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); - return res.status(200).send(data); + if (!isGzipped) { + data = await gzipP(data); } - } - }); - } - }, - ); - app.get('/:id.json', (req, res, next) => { + return res.status(200).send(data); + } + } + }); + } + }); + app.get('/:id.json', (req, res) => { const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); diff --git a/src/serve_font.js b/src/serve_font.js index 02f46dc05..d75390555 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -13,31 +13,29 @@ export const serve_font = async (options, allowedFonts) => { const existingFonts = {}; - app.get( - '/fonts/:fontstack/:range([\\d]+-[\\d]+).pbf', - async (req, res, next) => { - const fontstack = decodeURI(req.params.fontstack); - const range = req.params.range; - - try { - const concatenated = await getFontsPbf( - options.serveAllFonts ? null : allowedFonts, - fontPath, - fontstack, - range, - existingFonts, - ); - - res.header('Content-type', 'application/x-protobuf'); - res.header('Last-Modified', lastModified); - return res.send(concatenated); - } catch (err) { - res.status(400).header('Content-Type', 'text/plain').send(err); - } - }, - ); - - app.get('/fonts.json', (req, res, next) => { + app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + const fontstack = decodeURI(req.params.fontstack); + const range = req.params.range; + + try { + const concatenated = await getFontsPbf( + options.serveAllFonts ? null : allowedFonts, + fontPath, + fontstack, + range, + existingFonts, + ); + + res.header('Content-type', 'application/x-protobuf'); + res.header('Last-Modified', lastModified); + return res.send(concatenated); + } catch (err) { + console.error('Error serving font:', err); + return res.status(400).header('Content-Type', 'text/plain').send(err); + } + }); + + app.get('/fonts.json', (req, res) => { res.header('Content-type', 'application/json'); return res.send( Object.keys(options.serveAllFonts ? existingFonts : allowedFonts).sort(), diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 3e5c94eaa..65246f1bc 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -44,13 +44,43 @@ import fsp from 'node:fs/promises'; import { existsP, gunzipP } from './promises.js'; import { openMbTilesWrapper } from './mbtiles_wrapper.js'; -const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+.?\\d+)'; +const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d*\\.\\d+)'; + +const staticTypeRegex = new RegExp( + `^` + + `(?:` + + // Format 1: {lon},{lat},{zoom}[@{bearing}[,{pitch}]] + `(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN})` + + `(?:@(?${FLOAT_PATTERN})(?:,(?${FLOAT_PATTERN}))?)?` + + `|` + + // Format 2: {minx},{miny},{maxx},{maxy} + `(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN})` + + `|` + + // Format 3: auto + `(?auto)` + + `)` + + `$`, +); + const PATH_PATTERN = /^((fill|stroke|width)\:[^\|]+\|)*(enc:.+|-?\d+(\.\d*)?,-?\d+(\.\d*)?(\|-?\d+(\.\d*)?,-?\d+(\.\d*)?)+)/; const httpTester = /^https?:\/\//i; const mercator = new SphericalMercator(); -const getScale = (scale) => (scale || '@1x').slice(1, 2) | 0; + +const parseScale = (scale, maxScaleDigit = 9) => { + if (scale === undefined) { + return 1; + } + + // eslint-disable-next-line security/detect-non-literal-regexp + const regex = new RegExp(`^[2-${maxScaleDigit}]x$`); + if (!regex.test(scale)) { + return null; + } + + return parseInt(scale.slice(0, -1), 10); +}; mlgl.on('message', (e) => { if (e.severity === 'WARNING' || e.severity === 'ERROR') { @@ -555,307 +585,256 @@ let maxScaleFactor = 2; export const serve_rendered = { init: async (options, repo) => { maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); - let scalePattern = ''; - for (let i = 2; i <= maxScaleFactor; i++) { - scalePattern += i.toFixed(); - } - scalePattern = `@[${scalePattern}]x`; - const app = express().disable('x-powered-by'); app.get( - `/:id/(:tileSize(256|512)/)?:z(\\d+)/:x(\\d+)/:y(\\d+):scale(${scalePattern})?.:format([\\w]+)`, - (req, res, next) => { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } + `/:id{/:tileSize}/:z/:x/:y{@:scale}{.:format}`, + async (req, res, next) => { + try { + console.log(req.params); + if ( + req.params.z === 'static' || + (req.params.tileSize && + req.params.tileSize != 256 && + req.params.tileSize != 512) + ) { + //workaroud for /:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format} + next('route'); + } else { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(item.lastModified) <= new Date(modifiedSince)) { - return res.sendStatus(304); - } - } + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if (new Date(item.lastModified) <= new Date(modifiedSince)) { + return res.sendStatus(304); + } + } - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - const scale = getScale(req.params.scale); - const format = req.params.format; - const tileSize = parseInt(req.params.tileSize, 10) || 256; - - if ( - z < 0 || - x < 0 || - y < 0 || - z > 22 || - x >= Math.pow(2, z) || - y >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } + const z = req.params.z | 0; + const x = req.params.x | 0; + const y = req.params.y | 0; + const scale = parseScale(req.params.scale, maxScaleFactor); + const format = req.params.format; + const tileSize = parseInt(req.params.tileSize, 10) || 256; + if ( + scale == null || + z < 0 || + x < 0 || + y < 0 || + z > 22 || + x >= Math.pow(2, z) || + y >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } - const tileCenter = mercator.ll( - [ - ((x + 0.5) / (1 << z)) * (256 << z), - ((y + 0.5) / (1 << z)) * (256 << z), - ], - z, - ); + const tileCenter = mercator.ll( + [ + ((x + 0.5) / (1 << z)) * (256 << z), + ((y + 0.5) / (1 << z)) * (256 << z), + ], + z, + ); - // prettier-ignore - return respondImage( - options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, - ); + // prettier-ignore + return await respondImage( + options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, + ); + } + } catch (e) { + console.log(e); + next('route'); + } }, ); if (options.serveStaticMaps !== false) { - const staticPattern = `/:id/static/:raw(raw)?/%s/:width(\\d+)x:height(\\d+):scale(${scalePattern})?.:format([\\w]+)`; - - const centerPattern = util.format( - ':x(%s),:y(%s),:z(%s)(@:bearing(%s)(,:pitch(%s))?)?', - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - ); - app.get( - util.format(staticPattern, centerPattern), + `/:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format}`, async (req, res, next) => { try { const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const raw = req.params.raw; - const z = +req.params.z; - let x = +req.params.x; - let y = +req.params.y; - const bearing = +(req.params.bearing || '0'); - const pitch = +(req.params.pitch || '0'); - const w = req.params.width | 0; - const h = req.params.height | 0; - const scale = getScale(req.params.scale); + console.log(req.params); const format = req.params.format; - - if (z < 0) { - return res.status(404).send('Invalid zoom'); + const w = parseInt(req.params.width) || 512; + const h = parseInt(req.params.height) || 512; + const scale = parseScale(req.params.scale, maxScaleFactor); + let raw = req.params.raw !== undefined; + let type = req.params.type; + if (!type) { + //workaround for type when raw is not set + type = req.params.raw; + raw = false; } - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - if (transformer) { - const ll = transformer([x, y]); - x = ll[0]; - y = ll[1]; + if (!item || !type || !format || !scale) { + return res.sendStatus(404); } - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); - - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); - - // prettier-ignore - return respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } catch (e) { - next(e); - } - }, - ); - - const serveBounds = async (req, res, next) => { - try { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const raw = req.params.raw; - const bbox = [ - +req.params.minx, - +req.params.miny, - +req.params.maxx, - +req.params.maxy, - ]; - let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; - - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - if (transformer) { - const minCorner = transformer(bbox.slice(0, 2)); - const maxCorner = transformer(bbox.slice(2)); - bbox[0] = minCorner[0]; - bbox[1] = minCorner[1]; - bbox[2] = maxCorner[0]; - bbox[3] = maxCorner[1]; - center = transformer(center); - } + const staticTypeMatch = type.match(staticTypeRegex); + console.log(staticTypeMatch.groups); + if (staticTypeMatch.groups.lon) { + // Center Based Static Image + const z = staticTypeMatch.groups.zoom; + let x = staticTypeMatch.groups.lon; + let y = staticTypeMatch.groups.lat; + const bearing = staticTypeMatch.groups.bearing; + const pitch = staticTypeMatch.groups.pitch; + + if (z < 0) { + return res.status(404).send('Invalid zoom'); + } - const w = req.params.width | 0; - const h = req.params.height | 0; - const scale = getScale(req.params.scale); - const format = req.params.format; - - const z = calcZForBBox(bbox, w, h, req.query); - const x = center[0]; - const y = center[1]; - const bearing = 0; - const pitch = 0; - - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + const transformer = raw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); + if (transformer) { + const ll = transformer([x, y]); + x = ll[0]; + y = ll[1]; + } - // prettier-ignore - return respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } catch (e) { - next(e); - } - }; + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery( + req.query, + options, + transformer, + ); - const boundsPattern = util.format( - ':minx(%s),:miny(%s),:maxx(%s),:maxy(%s)', - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - ); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, + ); - app.get(util.format(staticPattern, boundsPattern), serveBounds); + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.minx) { + // Area Based Static Image + const bbox = [ + +staticTypeMatch.groups.minx, + +staticTypeMatch.groups.miny, + +staticTypeMatch.groups.maxx, + +staticTypeMatch.groups.maxx, + ]; + let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; + + const transformer = raw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + if (transformer) { + const minCorner = transformer(bbox.slice(0, 2)); + const maxCorner = transformer(bbox.slice(2)); + bbox[0] = minCorner[0]; + bbox[1] = minCorner[1]; + bbox[2] = maxCorner[0]; + bbox[3] = maxCorner[1]; + center = transformer(center); + } - app.get('/:id/static/', (req, res, next) => { - for (const key in req.query) { - req.query[key.toLowerCase()] = req.query[key]; - } - req.params.raw = true; - req.params.format = (req.query.format || 'image/png').split('/').pop(); - const bbox = (req.query.bbox || '').split(','); - req.params.minx = bbox[0]; - req.params.miny = bbox[1]; - req.params.maxx = bbox[2]; - req.params.maxy = bbox[3]; - req.params.width = req.query.width || '256'; - req.params.height = req.query.height || '256'; - if (req.query.scale) { - req.params.width /= req.query.scale; - req.params.height /= req.query.scale; - req.params.scale = `@${req.query.scale}`; - } + const z = calcZForBBox(bbox, w, h, req.query); + const x = center[0]; + const y = center[1]; + const bearing = 0; + const pitch = 0; + + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery( + req.query, + options, + transformer, + ); - return serveBounds(req, res, next); - }); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, + ); - const autoPattern = 'auto'; + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.auto) { + // Area Static Image + const bearing = 0; + const pitch = 0; + + const transformer = raw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery( + req.query, + options, + transformer, + ); - app.get( - util.format(staticPattern, autoPattern), - async (req, res, next) => { - try { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const raw = req.params.raw; - const w = req.params.width | 0; - const h = req.params.height | 0; - const bearing = 0; - const pitch = 0; - const scale = getScale(req.params.scale); - const format = req.params.format; + // Extract coordinates from markers + const markerCoordinates = []; + for (const marker of markers) { + markerCoordinates.push(marker.location); + } - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; + // Create array with coordinates from markers and path + const coords = [].concat(paths.flat()).concat(markerCoordinates); - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + // Check if we have at least one coordinate to calculate a bounding box + if (coords.length < 1) { + return res.status(400).send('No coordinates provided'); + } - // Extract coordinates from markers - const markerCoordinates = []; - for (const marker of markers) { - markerCoordinates.push(marker.location); - } + const bbox = [Infinity, Infinity, -Infinity, -Infinity]; + for (const pair of coords) { + bbox[0] = Math.min(bbox[0], pair[0]); + bbox[1] = Math.min(bbox[1], pair[1]); + bbox[2] = Math.max(bbox[2], pair[0]); + bbox[3] = Math.max(bbox[3], pair[1]); + } - // Create array with coordinates from markers and path - const coords = [].concat(paths.flat()).concat(markerCoordinates); + const bbox_ = mercator.convert(bbox, '900913'); + const center = mercator.inverse([ + (bbox_[0] + bbox_[2]) / 2, + (bbox_[1] + bbox_[3]) / 2, + ]); + + // Calculate zoom level + const maxZoom = parseFloat(req.query.maxzoom); + let z = calcZForBBox(bbox, w, h, req.query); + if (maxZoom > 0) { + z = Math.min(z, maxZoom); + } - // Check if we have at least one coordinate to calculate a bounding box - if (coords.length < 1) { - return res.status(400).send('No coordinates provided'); - } + const x = center[0]; + const y = center[1]; - const bbox = [Infinity, Infinity, -Infinity, -Infinity]; - for (const pair of coords) { - bbox[0] = Math.min(bbox[0], pair[0]); - bbox[1] = Math.min(bbox[1], pair[1]); - bbox[2] = Math.max(bbox[2], pair[0]); - bbox[3] = Math.max(bbox[3], pair[1]); - } + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, + ); - const bbox_ = mercator.convert(bbox, '900913'); - const center = mercator.inverse([ - (bbox_[0] + bbox_[2]) / 2, - (bbox_[1] + bbox_[3]) / 2, - ]); - - // Calculate zoom level - const maxZoom = parseFloat(req.query.maxzoom); - let z = calcZForBBox(bbox, w, h, req.query); - if (maxZoom > 0) { - z = Math.min(z, maxZoom); + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', + ); + } else { + return res.sendStatus(404); } - - const x = center[0]; - const y = center[1]; - - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); - - // prettier-ignore - return respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); } catch (e) { - next(e); + next('route'); } }, ); } - app.get('/(:tileSize(256|512)/)?:id.json', (req, res, next) => { + app.get('{/:tileSize}/:id.json', (req, res, next) => { const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); diff --git a/src/serve_style.js b/src/serve_style.js index 5d3b4699f..15f925064 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -10,9 +10,15 @@ import { validateStyleMin } from '@maplibre/maplibre-gl-style-spec'; import { fixUrl, allowedOptions } from './utils.js'; const httpTester = /^https?:\/\//i; -const allowedSpriteScales = allowedOptions(['', '@2x', '@3x']); const allowedSpriteFormats = allowedOptions(['png', 'json']); +const allowedSpriteScales = (scale) => { + if (!scale) return ''; // Default to 1 if no scale provided + const match = scale.match(/(\d+)x/); // Match one or more digits before 'x' + const parsedScale = match ? parseInt(match[1], 10) : 1; // Parse the number, or default to 1 if no match + return '@' + Math.min(parsedScale, 3) + 'x'; +}; + export const serve_style = { init: (options, repo) => { const app = express().disable('x-powered-by'); @@ -46,14 +52,18 @@ export const serve_style = { return res.send(styleJSON_); }); - app.get( - '/:id/sprite(/:spriteID)?:scale(@[23]x)?.:format([\\w]+)', - (req, res, next) => { - const { spriteID = 'default', id } = req.params; - const scale = allowedSpriteScales(req.params.scale) || ''; - const format = allowedSpriteFormats(req.params.format); - - if (format) { + app.get(`/:id/:sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + console.log(req.params); + const { spriteID = 'default', id, format } = req.params; + const scale = allowedSpriteScales(req.params.scale); + try { + if ( + !allowedSpriteFormats(format) || + ((id == 256 || id == 512) && format === 'json') + ) { + //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml + next('route'); + } else { const item = repo[id]; const sprite = item.spritePaths.find( (sprite) => sprite.id === spriteID, @@ -74,11 +84,12 @@ export const serve_style = { } else { return res.status(400).send('Bad Sprite ID or Scale'); } - } else { - return res.status(400).send('Bad Sprite Format'); } - }, - ); + } catch (e) { + console.log(e); + next('route'); + } + }); return app; }, diff --git a/src/server.js b/src/server.js index c554e83a4..56add035e 100644 --- a/src/server.js +++ b/src/server.js @@ -37,7 +37,7 @@ const serve_rendered = ( * * @param opts */ -function start(opts) { +async function start(opts) { console.log('Starting server'); const app = express().disable('x-powered-by'); @@ -73,7 +73,7 @@ function start(opts) { config = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch (e) { console.log('ERROR: Config file not found or invalid!'); - console.log(' See README.md for instructions and sample data.'); + console.log(' See README.md for instructions and sample data.'); process.exit(1); } } @@ -379,14 +379,14 @@ function start(opts) { return arr; }; - app.get('/(:tileSize(256|512)/)?rendered.json', (req, res, next) => { + app.get('{/:tileSize}/rendered.json', (req, res, next) => { const tileSize = parseInt(req.params.tileSize, 10) || undefined; res.send(addTileJSONs([], req, 'rendered', tileSize)); }); - app.get('/data.json', (req, res, next) => { + app.get('/data.json', (req, res) => { res.send(addTileJSONs([], req, 'data', undefined)); }); - app.get('/(:tileSize(256|512)/)?index.json', (req, res, next) => { + app.get('{/:tileSize}/index.json', (req, res, next) => { const tileSize = parseInt(req.params.tileSize, 10) || undefined; res.send( addTileJSONs( @@ -415,44 +415,38 @@ function start(opts) { templateFile = path.resolve(paths.root, options.frontPage); } } - startupPromises.push( - new Promise((resolve, reject) => { - fs.readFile(templateFile, (err, content) => { - if (err) { - err = new Error(`Template not found: ${err.message}`); - reject(err); - return; + try { + const content = fs.readFileSync(templateFile, 'utf-8'); + const compiled = handlebars.compile(content.toString()); + app.get(urlPath, (req, res) => { + console.log(`Serving template at path: ${urlPath}`); + let data = {}; + if (dataGetter) { + data = dataGetter(req); + if (!data) { + console.error(`Data getter for ${template} returned null`); + return res.status(404).send('Not found'); } - const compiled = handlebars.compile(content.toString()); - - app.use(urlPath, (req, res, next) => { - let data = {}; - if (dataGetter) { - data = dataGetter(req); - if (!data) { - return res.status(404).send('Not found'); - } - } - data['server_version'] = - `${packageJson.name} v${packageJson.version}`; - data['public_url'] = opts.publicUrl || '/'; - data['is_light'] = isLight; - data['key_query_part'] = req.query.key - ? `key=${encodeURIComponent(req.query.key)}&` - : ''; - data['key_query'] = req.query.key - ? `?key=${encodeURIComponent(req.query.key)}` - : ''; - if (template === 'wmts') res.set('Content-Type', 'text/xml'); - return res.status(200).send(compiled(data)); - }); - resolve(); - }); - }), - ); + } + data['server_version'] = `${packageJson.name} v${packageJson.version}`; + data['public_url'] = opts.publicUrl || '/'; + data['is_light'] = isLight; + data['key_query_part'] = req.query.key + ? `key=${encodeURIComponent(req.query.key)}&` + : ''; + data['key_query'] = req.query.key + ? `?key=${encodeURIComponent(req.query.key)}` + : ''; + if (template === 'wmts') res.set('Content-Type', 'text/xml'); + return res.status(200).send(compiled(data)); + }); + } catch (err) { + console.error(`Error reading template file: ${templateFile}`, err); + throw new Error(`Template not found: ${err.message}`); //throw an error so that the server doesnt start + } }; - serveTemplate('/$', 'index', (req) => { + serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { let style = { @@ -542,7 +536,7 @@ function start(opts) { }; }); - serveTemplate('/styles/:id/$', 'viewer', (req) => { + serveTemplate('/styles/:id/', 'viewer', (req) => { const { id } = req.params; const style = clone(((serving.styles || {})[id] || {}).styleJSON); @@ -559,11 +553,6 @@ function start(opts) { }; }); - /* - app.use('/rendered/:id/$', function(req, res, next) { - return res.redirect(301, '/styles/' + req.params.id + '/'); - }); - */ serveTemplate('/styles/:id/wmts.xml', 'wmts', (req) => { const { id } = req.params; const wmts = clone((serving.styles || {})[id]); @@ -595,7 +584,7 @@ function start(opts) { }; }); - serveTemplate('/data/:id/$', 'data', (req) => { + serveTemplate('/data/:id/', 'data', (req) => { const { id } = req.params; const data = serving.data[id]; @@ -616,7 +605,7 @@ function start(opts) { startupComplete = true; }); - app.get('/health', (req, res, next) => { + app.get('/health', (req, res) => { if (startupComplete) { return res.status(200).send('OK'); } else { @@ -659,8 +648,8 @@ function stopGracefully(signal) { * * @param opts */ -export function server(opts) { - const running = start(opts); +export async function server(opts) { + const running = await start(opts); running.startupPromise.catch((err) => { console.error(err.message); diff --git a/test/setup.js b/test/setup.js index 34fba6707..1852a195c 100644 --- a/test/setup.js +++ b/test/setup.js @@ -7,10 +7,10 @@ import { server } from '../src/server.js'; global.expect = expect; global.supertest = supertest; -before(function () { +before(async function () { console.log('global setup'); process.chdir('test_data'); - const running = server({ + const running = await server({ configPath: 'config.json', port: 8888, publicUrl: '/test/', From b72f6621065ec30eca7c88134aff37dac9732c3e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 01:56:15 -0500 Subject: [PATCH 002/104] try to fix https://github.com/maptiler/tileserver-gl/issues/1411 Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 65246f1bc..05d14ceb8 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -133,23 +133,31 @@ function createEmptyResponse(format, color, callback) { } // create an "empty" response image - color = new Color(color); - const array = color.array(); - const channels = array.length === 4 && format !== 'jpeg' ? 4 : 3; - sharp(Buffer.from(array), { - raw: { - width: 1, - height: 1, - channels, - }, - }) - .toFormat(format) - .toBuffer((err, buffer, info) => { - if (!err) { + try { + color = new Color(color); + const array = color.array(); + const channels = array.length === 4 && format !== 'jpeg' ? 4 : 3; + sharp(Buffer.from(array), { + raw: { + width: 1, + height: 1, + channels, + }, + }) + .toFormat(format) + .toBuffer((err, buffer, info) => { + if (err) { + console.error('Error creating image with Sharp:', err); + callback(err, null); + return; + } cachedEmptyResponses[cacheKey] = buffer; - } - callback(null, { data: buffer }); - }); + callback(null, { data: buffer }); + }); + } catch (error) { + console.error('Error during image processing setup:', error); + callback(error, null); + } } /** From c7377e82d55bc6b7f8140f66ed7c17d37edf4955 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 02:28:42 -0500 Subject: [PATCH 003/104] cleanup server.js Co-Authored-By: Andrew Calcutt --- src/server.js | 64 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/server.js b/src/server.js index 56add035e..9f6d71401 100644 --- a/src/server.js +++ b/src/server.js @@ -34,8 +34,9 @@ const serve_rendered = ( ).serve_rendered; /** - * - * @param opts + * Starts the server. + * @param {object} opts - Configuration options for the server. + * @returns {Promise} - A promise that resolves to the server object. */ async function start(opts) { console.log('Starting server'); @@ -116,8 +117,9 @@ async function start(opts) { * Recursively get all files within a directory. * Inspired by https://stackoverflow.com/a/45130990/10133863 * @param {string} directory Absolute path to a directory to get files from. + * @returns {Promise} - A promise that resolves to an array of file paths relative to the icon directory. */ - const getFiles = async (directory) => { + async function getFiles(directory) { // Fetch all entries of the directory and attach type information const dirEntries = await fs.promises.readdir(directory, { withFileTypes: true, @@ -136,7 +138,7 @@ async function start(opts) { // Flatten the list of files to a single array return files.flat(); - }; + } // Load all available icons into a settings object startupPromises.push( @@ -169,8 +171,15 @@ async function start(opts) { }), ); } - - const addStyle = (id, item, allowMoreData, reportFonts) => { + /** + * Adds a style to the server. + * @param {string} id - The ID of the style. + * @param {object} item - The style configuration object. + * @param {boolean} allowMoreData - Whether to allow adding more data sources. + * @param {boolean} reportFonts - Whether to report fonts. + * @returns {void} + */ + function addStyle(id, item, allowMoreData, reportFonts) { let success = true; if (item.serve_data !== false) { success = serve_style.add( @@ -261,7 +270,7 @@ async function start(opts) { item.serve_rendered = false; } } - }; + } for (const id of Object.keys(config.styles || {})) { const item = config.styles[id]; @@ -272,13 +281,11 @@ async function start(opts) { addStyle(id, item, true, true); } - startupPromises.push( serve_font(options, serving.fonts).then((sub) => { app.use('/', sub); }), ); - for (const id of Object.keys(data)) { const item = data[id]; const fileType = Object.keys(data[id])[0]; @@ -288,12 +295,10 @@ async function start(opts) { ); continue; } - startupPromises.push( serve_data.add(options, serving.data, item, id, opts.publicUrl), ); } - if (options.serveAllStyles) { fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => { if (err) { @@ -333,7 +338,6 @@ async function start(opts) { } }); } - app.get('/styles.json', (req, res, next) => { const result = []; const query = req.query.key @@ -354,7 +358,15 @@ async function start(opts) { res.send(result); }); - const addTileJSONs = (arr, req, type, tileSize) => { + /** + * Adds TileJSON metadata to an array. + * @param {Array} arr - The array to add TileJSONs to + * @param {object} req - The express request object. + * @param {string} type - The type of resource + * @param {number} tileSize - The tile size. + * @returns {Array} - An array of TileJSON objects. + */ + function addTileJSONs(arr, req, type, tileSize) { for (const id of Object.keys(serving[type])) { const info = clone(serving[type][id].tileJSON); let path = ''; @@ -377,7 +389,7 @@ async function start(opts) { arr.push(info); } return arr; - }; + } app.get('{/:tileSize}/rendered.json', (req, res, next) => { const tileSize = parseInt(req.params.tileSize, 10) || undefined; @@ -403,7 +415,14 @@ async function start(opts) { app.use('/', express.static(path.join(__dirname, '../public/resources'))); const templates = path.join(__dirname, '../public/templates'); - const serveTemplate = (urlPath, template, dataGetter) => { + /** + * Serves a Handlebars template. + * @param {string} urlPath - The URL path to serve the template at + * @param {string} template - The name of the template file + * @param {Function} dataGetter - A function to get data to be passed to the template. + * @returns {void} + */ + function serveTemplate(urlPath, template, dataGetter) { let templateFile = `${templates}/${template}.tmpl`; if (template === 'index') { if (options.frontPage === false) { @@ -444,8 +463,7 @@ async function start(opts) { console.error(`Error reading template file: ${templateFile}`, err); throw new Error(`Template not found: ${err.message}`); //throw an error so that the server doesnt start } - }; - + } serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { @@ -478,7 +496,6 @@ async function start(opts) { styles[id] = style; } - let datas = {}; for (const id of Object.keys(serving.data || {})) { let data = Object.assign({}, serving.data[id]); @@ -526,10 +543,8 @@ async function start(opts) { } data.formatted_filesize = `${size.toFixed(2)} ${suffix}`; } - datas[id] = data; } - return { styles: Object.keys(styles).length ? styles : null, data: Object.keys(datas).length ? datas : null, @@ -543,7 +558,6 @@ async function start(opts) { if (!style) { return null; } - return { ...style, id, @@ -591,7 +605,6 @@ async function start(opts) { if (!data) { return null; } - return { ...data, id, @@ -638,6 +651,7 @@ async function start(opts) { /** * Stop the server gracefully * @param {string} signal Name of the received signal + * @returns {void} */ function stopGracefully(signal) { console.log(`Caught signal ${signal}, stopping gracefully`); @@ -645,8 +659,9 @@ function stopGracefully(signal) { } /** - * - * @param opts + * Starts and manages the server + * @param {object} opts - Configuration options for the server. + * @returns {Promise} - A promise that resolves to the running server */ export async function server(opts) { const running = await start(opts); @@ -669,6 +684,5 @@ export async function server(opts) { running.app = restarted.app; }); }); - return running; } From 2e74bc7b4ac03c299a34d6ff72af7ec57c1e9e3e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 02:34:39 -0500 Subject: [PATCH 004/104] cleanup serve_font.js Co-Authored-By: Andrew Calcutt --- src/serve_font.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index d75390555..30f1fc8dc 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -4,7 +4,13 @@ import express from 'express'; import { getFontsPbf, listFonts } from './utils.js'; -export const serve_font = async (options, allowedFonts) => { +/** + * Initializes and returns an Express app that serves font files. + * @param {object} options - Configuration options for the server. + * @param {object} allowedFonts - An object containing allowed fonts. + * @returns {Promise} - A promise that resolves to the Express app. + */ +export async function serve_font(options, allowedFonts) { const app = express().disable('x-powered-by'); const lastModified = new Date().toUTCString(); @@ -45,4 +51,4 @@ export const serve_font = async (options, allowedFonts) => { const fonts = await listFonts(options.paths.fonts); Object.assign(existingFonts, fonts); return app; -}; +} From 61e81e0e610368ec80332aa8cff143025210fe86 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 02:58:59 -0500 Subject: [PATCH 005/104] cleanup sever_rendered.js Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 123 +++++++++++++++++++++++++++++++++--------- 1 file changed, 98 insertions(+), 25 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 05d14ceb8..f3b5d94e5 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -68,7 +68,13 @@ const httpTester = /^https?:\/\//i; const mercator = new SphericalMercator(); -const parseScale = (scale, maxScaleDigit = 9) => { +/** + * Parses a scale string to a number. + * @param {string} scale The scale string (e.g., '2x', '4x'). + * @param {number} maxScaleDigit Maximum allowed scale digit. + * @returns {number|null} The parsed scale as a number or null if invalid. + */ +function parseScale(scale, maxScaleDigit = 9) { if (scale === undefined) { return 1; } @@ -80,7 +86,7 @@ const parseScale = (scale, maxScaleDigit = 9) => { } return parseInt(scale.slice(0, -1), 10); -}; +} mlgl.on('message', (e) => { if (e.severity === 'WARNING' || e.severity === 'ERROR') { @@ -111,6 +117,7 @@ const cachedEmptyResponses = { * @param {string} format The format (a sharp format or 'pbf'). * @param {string} color The background color (or empty string for transparent). * @param {Function} callback The mlgl callback. + * @returns {void} */ function createEmptyResponse(format, color, callback) { if (!format || format === 'pbf') { @@ -163,11 +170,12 @@ function createEmptyResponse(format, color, callback) { /** * Parses coordinate pair provided to pair of floats and ensures the resulting * pair is a longitude/latitude combination depending on lnglat query parameter. - * @param {List} coordinatePair Coordinate pair. + * @param {Array} coordinatePair Coordinate pair. * @param coordinates * @param {object} query Request query parameters. + * @returns {Array|null} Parsed coordinate pair as [longitude, latitude] or null if invalid */ -const parseCoordinatePair = (coordinates, query) => { +function parseCoordinatePair(coordinates, query) { const firstCoordinate = parseFloat(coordinates[0]); const secondCoordinate = parseFloat(coordinates[1]); @@ -183,15 +191,16 @@ const parseCoordinatePair = (coordinates, query) => { } return [firstCoordinate, secondCoordinate]; -}; +} /** * Parses a coordinate pair from query arguments and optionally transforms it. - * @param {List} coordinatePair Coordinate pair. + * @param {Array} coordinatePair Coordinate pair. * @param {object} query Request query parameters. * @param {Function} transformer Optional transform function. + * @returns {Array|null} Transformed coordinate pair or null if invalid. */ -const parseCoordinates = (coordinatePair, query, transformer) => { +function parseCoordinates(coordinatePair, query, transformer) { const parsedCoordinates = parseCoordinatePair(coordinatePair, query); // Transform coordinates @@ -200,14 +209,15 @@ const parseCoordinates = (coordinatePair, query, transformer) => { } return parsedCoordinates; -}; +} /** * Parses paths provided via query into a list of path objects. * @param {object} query Request query parameters. * @param {Function} transformer Optional transform function. + * @returns {Array>>} Array of paths. */ -const extractPathsFromQuery = (query, transformer) => { +function extractPathsFromQuery(query, transformer) { // Initiate paths array const paths = []; // Return an empty list if no paths have been provided @@ -259,17 +269,18 @@ const extractPathsFromQuery = (query, transformer) => { } } return paths; -}; +} /** * Parses marker options provided via query and sets corresponding attributes * on marker object. * Options adhere to the following format * [optionName]:[optionValue] - * @param {List[String]} optionsList List of option strings. + * @param {Array} optionsList List of option strings. * @param {object} marker Marker object to configure. + * @returns {void} */ -const parseMarkerOptions = (optionsList, marker) => { +function parseMarkerOptions(optionsList, marker) { for (const options of optionsList) { const optionParts = options.split(':'); // Ensure we got an option name and value @@ -296,15 +307,16 @@ const parseMarkerOptions = (optionsList, marker) => { break; } } -}; +} /** * Parses markers provided via query into a list of marker objects. * @param {object} query Request query parameters. * @param {object} options Configuration options. * @param {Function} transformer Optional transform function. + * @returns {Array} An array of marker objects. */ -const extractMarkersFromQuery = (query, options, transformer) => { +function extractMarkersFromQuery(query, options, transformer) { // Return an empty list if no markers have been provided if (!query.marker) { return []; @@ -380,9 +392,16 @@ const extractMarkersFromQuery = (query, options, transformer) => { markers.push(marker); } return markers; -}; - -const calcZForBBox = (bbox, w, h, query) => { +} +/** + * Calculates the zoom level for a given bounding box. + * @param {Array} bbox Bounding box as [minx, miny, maxx, maxy]. + * @param {number} w Width of the image. + * @param {number} h Height of the image. + * @param {object} query Request query parameters. + * @returns {number} Calculated zoom level. + */ +function calcZForBBox(bbox, w, h, query) { let z = 25; const padding = query.padding !== undefined ? parseFloat(query.padding) : 0.1; @@ -401,9 +420,27 @@ const calcZForBBox = (bbox, w, h, query) => { z = Math.max(Math.log(Math.max(w, h) / 256) / Math.LN2, Math.min(25, z)); return z; -}; +} -const respondImage = ( +/** + * Responds with an image. + * @param {object} options Configuration options. + * @param {object} item Item object containing map and other information. + * @param {number} z Zoom level. + * @param {number} lon Longitude of the center. + * @param {number} lat Latitude of the center. + * @param {number} bearing Map bearing. + * @param {number} pitch Map pitch. + * @param {number} width Width of the image. + * @param {number} height Height of the image. + * @param {number} scale Scale factor. + * @param {string} format Image format. + * @param {object} res Express response object. + * @param {Buffer|null} overlay Optional overlay image. + * @param {string} mode Rendering mode ('tile' or 'static'). + * @returns {Promise} + */ +const respondImage = async ( options, item, z, @@ -451,7 +488,7 @@ const respondImage = ( } else { pool = item.map.renderersStatic[scale]; } - pool.acquire((err, renderer) => { + pool.acquire(async (err, renderer) => { // For 512px tiles, use the actual maplibre-native zoom. For 256px tiles, use zoom - 1 let mlglZ; if (width === 512) { @@ -591,7 +628,13 @@ const existingFonts = {}; let maxScaleFactor = 2; export const serve_rendered = { - init: async (options, repo) => { + /** + * Initializes the serve_rendered module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {Promise} A promise that resolves to the Express app. + */ + init: async function (options, repo) { maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); const app = express().disable('x-powered-by'); @@ -650,8 +693,8 @@ export const serve_rendered = { // prettier-ignore return await respondImage( - options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, - ); + options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, + ); } } catch (e) { console.log(e); @@ -864,7 +907,17 @@ export const serve_rendered = { Object.assign(existingFonts, fonts); return app; }, - add: async (options, repo, params, id, publicUrl, dataResolver) => { + /** + * Adds a new item to the repository. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @param {object} params Parameters object. + * @param {string} id ID of the item. + * @param {string} publicUrl Public URL. + * @param {Function} dataResolver Function to resolve data. + * @returns {Promise} + */ + add: async function (options, repo, params, id, publicUrl, dataResolver) { const map = { renderers: [], renderersStatic: [], @@ -873,7 +926,21 @@ export const serve_rendered = { }; let styleJSON; + /** + * Creates a pool of renderers. + * @param {number} ratio Pixel ratio + * @param {string} mode Rendering mode ('tile' or 'static'). + * @param {number} min Minimum pool size. + * @param {number} max Maximum pool size. + * @returns {object} The created pool + */ const createPool = (ratio, mode, min, max) => { + /** + * Creates a renderer + * @param {number} ratio Pixel ratio + * @param {Function} createCallback Function that returns the renderer when created + * @returns {void} + */ const createRenderer = (ratio, createCallback) => { const renderer = new mlgl.Map({ mode, @@ -1278,7 +1345,13 @@ export const serve_rendered = { ); } }, - remove: (repo, id) => { + /** + * Removes an item from the repository. + * @param {object} repo Repository object. + * @param {string} id ID of the item to remove. + * @returns {void} + */ + remove: function (repo, id) { const item = repo[id]; if (item) { item.map.renderers.forEach((pool) => { From a0fb3680d0026f3147994e94b2a44d6e2ed3a76f Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 03:09:09 -0500 Subject: [PATCH 006/104] cleanup server_data.js Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index d9bea6871..6cff9c3f0 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -18,7 +18,13 @@ import { gunzipP, gzipP } from './promises.js'; import { openMbTilesWrapper } from './mbtiles_wrapper.js'; export const serve_data = { - init: (options, repo) => { + /** + * Initializes the serve_data module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {express.Application} The initialized Express application. + */ + init: function (options, repo) { const app = express().disable('x-powered-by'); app.get('/:id/:z/:x/:y.:format', async (req, res) => { @@ -181,7 +187,16 @@ export const serve_data = { return app; }, - add: async (options, repo, params, id, publicUrl) => { + /** + * Adds a new data source to the repository. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @param {object} params Parameters object. + * @param {string} id ID of the data source. + * @param {string} publicUrl Public URL of the data. + * @returns {Promise} + */ + add: async function (options, repo, params, id, publicUrl) { let inputFile; let inputType; if (params.pmtiles) { From b6382085b74724aa4c415ab4b429b82af7fca8ae Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 03:15:54 -0500 Subject: [PATCH 007/104] cleanup serve_style Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 15f925064..ec8a0b448 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -12,15 +12,26 @@ import { fixUrl, allowedOptions } from './utils.js'; const httpTester = /^https?:\/\//i; const allowedSpriteFormats = allowedOptions(['png', 'json']); -const allowedSpriteScales = (scale) => { +/** + * Checks and formats sprite scale + * @param {string} scale string containing the scale + * @returns {string} formated string for the scale or empty string if scale is invalid + */ +function allowedSpriteScales(scale) { if (!scale) return ''; // Default to 1 if no scale provided const match = scale.match(/(\d+)x/); // Match one or more digits before 'x' const parsedScale = match ? parseInt(match[1], 10) : 1; // Parse the number, or default to 1 if no match return '@' + Math.min(parsedScale, 3) + 'x'; -}; +} export const serve_style = { - init: (options, repo) => { + /** + * Initializes the serve_style module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {express.Application} The initialized Express application. + */ + init: function (options, repo) { const app = express().disable('x-powered-by'); app.get('/:id/style.json', (req, res, next) => { @@ -93,10 +104,35 @@ export const serve_style = { return app; }, - remove: (repo, id) => { + /** + * Removes an item from the repository. + * @param {object} repo Repository object. + * @param {string} id ID of the item to remove. + * @returns {void} + */ + remove: function (repo, id) { delete repo[id]; }, - add: (options, repo, params, id, publicUrl, reportTiles, reportFont) => { + /** + * Adds a new style to the repository. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @param {object} params Parameters object containing style path + * @param {string} id ID of the style. + * @param {string} publicUrl Public URL of the data. + * @param {Function} reportTiles Function for reporting tile sources. + * @param {Function} reportFont Function for reporting font usage + * @returns {boolean} true if add is succesful + */ + add: function ( + options, + repo, + params, + id, + publicUrl, + reportTiles, + reportFont, + ) { const styleFile = path.resolve(options.paths.styles, params.style); let styleFileData; From d635d3ca326afe67fca2d3e29ed687b733b7889d Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 03:19:21 -0500 Subject: [PATCH 008/104] Update serve_style.js Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index ec8a0b448..d0c2e3b82 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -18,9 +18,9 @@ const allowedSpriteFormats = allowedOptions(['png', 'json']); * @returns {string} formated string for the scale or empty string if scale is invalid */ function allowedSpriteScales(scale) { - if (!scale) return ''; // Default to 1 if no scale provided - const match = scale.match(/(\d+)x/); // Match one or more digits before 'x' - const parsedScale = match ? parseInt(match[1], 10) : 1; // Parse the number, or default to 1 if no match + if (!scale) return ''; + const match = scale.match(/(\d+)x/); + const parsedScale = match ? parseInt(match[1], 10) : 1; return '@' + Math.min(parsedScale, 3) + 'x'; } From c72d6f580c5e92181b2ceb7132d09521c94f63f3 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 11:39:38 -0500 Subject: [PATCH 009/104] Move UV_THREADPOOL_SIZE to main thred Co-Authored-By: Andrew Calcutt --- src/main.js | 6 ++++++ src/server.js | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main.js b/src/main.js index 7523aa937..b1f14a239 100644 --- a/src/main.js +++ b/src/main.js @@ -1,6 +1,12 @@ #!/usr/bin/env node 'use strict'; +import os from 'os'; + +const envSize = parseInt(process.env.UV_THREADPOOL_SIZE, 10); +process.env.UV_THREADPOOL_SIZE = Math.ceil( + Math.max(4, isNaN(envSize) ? os.cpus().length * 1.5 : envSize), +); import fs from 'node:fs'; import fsp from 'node:fs/promises'; diff --git a/src/server.js b/src/server.js index 9f6d71401..3da2ffb04 100644 --- a/src/server.js +++ b/src/server.js @@ -1,9 +1,6 @@ #!/usr/bin/env node 'use strict'; -import os from 'os'; -process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, os.cpus().length * 1.5)); - import fs from 'node:fs'; import path from 'path'; import fnv1a from '@sindresorhus/fnv1a'; From 70d6986a9ebd1427d21c97094b93d841362133fa Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 17:05:01 -0500 Subject: [PATCH 010/104] cleanup utils.js Co-Authored-By: Andrew Calcutt --- src/utils.js | 110 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 29 deletions(-) diff --git a/src/utils.js b/src/utils.js index 14e587145..5d01b5f8f 100644 --- a/src/utils.js +++ b/src/utils.js @@ -9,9 +9,10 @@ import { existsP } from './promises.js'; /** * Restrict user input to an allowed set of options. - * @param opts - * @param root0 - * @param root0.defaultValue + * @param {string[]} opts - An array of allowed option strings. + * @param {object} [config] - Optional configuration object. + * @param {string} [config.defaultValue] - The default value to return if input doesn't match. + * @returns {function(string): string} - A function that takes a value and returns it if valid or a default. */ export function allowedOptions(opts, { defaultValue } = {}) { const values = Object.fromEntries(opts.map((key) => [key, key])); @@ -19,10 +20,11 @@ export function allowedOptions(opts, { defaultValue } = {}) { } /** - * Replace local:// urls with public http(s):// urls - * @param req - * @param url - * @param publicUrl + * Replaces local:// URLs with public http(s):// URLs. + * @param {object} req - Express request object. + * @param {string} url - The URL string to fix. + * @param {string} publicUrl - The public URL prefix to use for replacements. + * @returns {string} - The fixed URL string. */ export function fixUrl(req, url, publicUrl) { if (!url || typeof url !== 'string' || url.indexOf('local://') !== 0) { @@ -40,12 +42,11 @@ export function fixUrl(req, url, publicUrl) { } /** - * Generate new URL object - * @param req - * @params {object} req - Express request - * @returns {URL} object + * Generates a new URL object from the Express request. + * @param {object} req - Express request object. + * @returns {URL} - URL object with correct host and optionally path. */ -const getUrlObject = (req) => { +function getUrlObject(req) { const urlObject = new URL(`${req.protocol}://${req.headers.host}/`); // support overriding hostname by sending X-Forwarded-Host http header urlObject.hostname = req.hostname; @@ -56,16 +57,33 @@ const getUrlObject = (req) => { urlObject.pathname = path.posix.join(xForwardedPath, urlObject.pathname); } return urlObject; -}; +} -export const getPublicUrl = (publicUrl, req) => { +/** + * Gets the public URL, either from a provided publicUrl or generated from the request. + * @param {string} publicUrl - The optional public URL to use. + * @param {object} req - The Express request object. + * @returns {string} - The final public URL string. + */ +export function getPublicUrl(publicUrl, req) { if (publicUrl) { return publicUrl; } return getUrlObject(req).toString(); -}; +} -export const getTileUrls = ( +/** + * Generates an array of tile URLs based on given parameters. + * @param {object} req - Express request object. + * @param {string | string[]} domains - Domain(s) to use for tile URLs. + * @param {string} path - The base path for the tiles. + * @param {number} [tileSize] - The size of the tile (optional). + * @param {string} format - The format of the tiles (e.g., 'png', 'jpg'). + * @param {string} publicUrl - The public URL to use (if not using domains). + * @param {object} [aliases] - Aliases for format extensions. + * @returns {string[]} An array of tile URL strings. + */ +export function getTileUrls( req, domains, path, @@ -73,7 +91,7 @@ export const getTileUrls = ( format, publicUrl, aliases, -) => { +) { const urlObject = getUrlObject(req); if (domains) { if (domains.constructor === String && domains.length > 0) { @@ -132,9 +150,14 @@ export const getTileUrls = ( } return uris; -}; +} -export const fixTileJSONCenter = (tileJSON) => { +/** + * Fixes the center in the tileJSON if no center is available. + * @param {object} tileJSON - The tileJSON object to process. + * @returns {void} + */ +export function fixTileJSONCenter(tileJSON) { if (tileJSON.bounds && !tileJSON.center) { const fitWidth = 1024; const tiles = fitWidth / 256; @@ -147,10 +170,19 @@ export const fixTileJSONCenter = (tileJSON) => { ), ]; } -}; +} -const getFontPbf = (allowedFonts, fontPath, name, range, fallbacks) => - new Promise((resolve, reject) => { +/** + * Retrieves font data for a given font and range. + * @param {object} allowedFonts - An object of allowed fonts. + * @param {string} fontPath - The path to the font directory. + * @param {string} name - The name of the font. + * @param {string} range - The range (e.g., '0-255') of the font to load. + * @param {object} [fallbacks] - Optional fallback font list. + * @returns {Promise} A promise that resolves with the font data Buffer or rejects with an error. + */ +function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { + return new Promise((resolve, reject) => { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { const filename = path.join(fontPath, name, `${range}.pbf`); if (!fallbacks) { @@ -192,14 +224,24 @@ const getFontPbf = (allowedFonts, fontPath, name, range, fallbacks) => reject(`Font not allowed: ${name}`); } }); +} -export const getFontsPbf = async ( +/** + * Combines multiple font pbf buffers into one. + * @param {object} allowedFonts - An object of allowed fonts. + * @param {string} fontPath - The path to the font directory. + * @param {string} names - Comma-separated font names. + * @param {string} range - The range of the font (e.g., '0-255'). + * @param {object} [fallbacks] - Fallback font list. + * @returns {Promise} - A promise that resolves to the combined font data buffer. + */ +export async function getFontsPbf( allowedFonts, fontPath, names, range, fallbacks, -) => { +) { const fonts = names.split(','); const queue = []; for (const font of fonts) { @@ -216,9 +258,14 @@ export const getFontsPbf = async ( const combined = combine(await Promise.all(queue), names); return Buffer.from(combined.buffer, 0, combined.buffer.length); -}; +} -export const listFonts = async (fontPath) => { +/** + * Lists available fonts in a given font directory. + * @param {string} fontPath - The path to the font directory. + * @returns {Promise} - Promise that resolves with an object where keys are the font names. + */ +export async function listFonts(fontPath) { const existingFonts = {}; const files = await fsPromises.readdir(fontPath); @@ -233,9 +280,14 @@ export const listFonts = async (fontPath) => { } return existingFonts; -}; +} -export const isValidHttpUrl = (string) => { +/** + * Checks if a string is a valid HTTP or HTTPS URL. + * @param {string} string - The string to validate. + * @returns {boolean} True if the string is a valid HTTP/HTTPS URL, false otherwise. + */ +export function isValidHttpUrl(string) { let url; try { @@ -245,4 +297,4 @@ export const isValidHttpUrl = (string) => { } return url.protocol === 'http:' || url.protocol === 'https:'; -}; +} From e1460fbf67e1dc7d8ff73d4df121943fa6145235 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 22:15:33 -0500 Subject: [PATCH 011/104] Use common app.get for images and static images Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 508 +++++++++++++++++++++++------------------- 1 file changed, 273 insertions(+), 235 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index f3b5d94e5..39649477b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -624,266 +624,304 @@ const respondImage = async ( }); }; -const existingFonts = {}; -let maxScaleFactor = 2; - -export const serve_rendered = { - /** - * Initializes the serve_rendered module. - * @param {object} options Configuration options. - * @param {object} repo Repository object. - * @returns {Promise} A promise that resolves to the Express app. - */ - init: async function (options, repo) { - maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); - const app = express().disable('x-powered-by'); - - app.get( - `/:id{/:tileSize}/:z/:x/:y{@:scale}{.:format}`, - async (req, res, next) => { - try { - console.log(req.params); - if ( - req.params.z === 'static' || - (req.params.tileSize && - req.params.tileSize != 256 && - req.params.tileSize != 512) - ) { - //workaroud for /:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format} - next('route'); - } else { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } +/** + * Handles requests for tile images. + * @param {object} options - Configuration options for the server. + * @param {object} repo - The repository object holding style data. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {Function} next - Express next middleware function. + * @param {number} maxScaleFactor - The maximum scale factor allowed. + * @returns {Promise} + */ +async function handleTileRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, +) { + const { + id, + p2: zParam, + p3: xParam, + p4: yParam, + scale: scaleParam, + format, + p1: tileSize, + } = req.params; + const item = repo[id]; + if (!item) { + return res.sendStatus(404); + } - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(item.lastModified) <= new Date(modifiedSince)) { - return res.sendStatus(304); - } - } + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if (new Date(item.lastModified) <= new Date(modifiedSince)) { + return res.sendStatus(304); + } + } + const z = parseFloat(zParam) | 0; + const x = parseFloat(xParam) | 0; + const y = parseFloat(yParam) | 0; + const scale = parseScale(scaleParam, maxScaleFactor); + const parsedTileSize = parseInt(tileSize, 10) || 256; + if ( + scale == null || + z < 0 || + x < 0 || + y < 0 || + z > 22 || + x >= Math.pow(2, z) || + y >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - const scale = parseScale(req.params.scale, maxScaleFactor); - const format = req.params.format; - const tileSize = parseInt(req.params.tileSize, 10) || 256; - if ( - scale == null || - z < 0 || - x < 0 || - y < 0 || - z > 22 || - x >= Math.pow(2, z) || - y >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } + const tileCenter = mercator.ll( + [((x + 0.5) / (1 << z)) * (256 << z), ((y + 0.5) / (1 << z)) * (256 << z)], + z, + ); - const tileCenter = mercator.ll( - [ - ((x + 0.5) / (1 << z)) * (256 << z), - ((y + 0.5) / (1 << z)) * (256 << z), - ], - z, - ); - - // prettier-ignore - return await respondImage( - options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, - ); - } - } catch (e) { - console.log(e); - next('route'); - } - }, - ); + // prettier-ignore + return await respondImage( + options, item, z, tileCenter[0], tileCenter[1], 0, 0, parsedTileSize, parsedTileSize, scale, format, res, + ); +} - if (options.serveStaticMaps !== false) { - app.get( - `/:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format}`, - async (req, res, next) => { - try { - const item = repo[req.params.id]; - console.log(req.params); - const format = req.params.format; - const w = parseInt(req.params.width) || 512; - const h = parseInt(req.params.height) || 512; - const scale = parseScale(req.params.scale, maxScaleFactor); - let raw = req.params.raw !== undefined; - let type = req.params.type; - if (!type) { - //workaround for type when raw is not set - type = req.params.raw; - raw = false; - } +/** + * Handles requests for static map images. + * @param {object} options - Configuration options for the server. + * @param {object} repo - The repository object holding style data. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {Function} next - Express next middleware function. + * @param {number} maxScaleFactor - The maximum scale factor allowed. + * @returns {Promise} + */ +async function handleStaticRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, +) { + const { + id, + scale: scaleParam, + format, + p2: raw, + p3: type, + p4: width, + p5: height, + } = req.params; + const item = repo[id]; + const parsedWidth = parseInt(width) || 512; + const parsedHeight = parseInt(height) || 512; + const scale = parseScale(scaleParam, maxScaleFactor); + let isRaw = raw !== undefined; + let staticType = type; + + if (!staticType) { + //workaround for type when raw is not set + staticType = raw; + isRaw = false; + } - if (!item || !type || !format || !scale) { - return res.sendStatus(404); - } + if (!item || !staticType || !format || !scale) { + return res.sendStatus(404); + } - const staticTypeMatch = type.match(staticTypeRegex); - console.log(staticTypeMatch.groups); - if (staticTypeMatch.groups.lon) { - // Center Based Static Image - const z = staticTypeMatch.groups.zoom; - let x = staticTypeMatch.groups.lon; - let y = staticTypeMatch.groups.lat; - const bearing = staticTypeMatch.groups.bearing; - const pitch = staticTypeMatch.groups.pitch; - - if (z < 0) { - return res.status(404).send('Invalid zoom'); - } + const staticTypeMatch = staticType.match(staticTypeRegex); + console.log(staticTypeMatch); + if (staticTypeMatch.groups.lon) { + // Center Based Static Image + const z = parseFloat(staticTypeMatch.groups.zoom) || 0; + let x = parseFloat(staticTypeMatch.groups.lon) || 0; + let y = parseFloat(staticTypeMatch.groups.lat) || 0; + const bearing = parseFloat(staticTypeMatch.groups.bearing) || 0; + const pitch = parseInt(staticTypeMatch.groups.pitch) || 0; + if (z < 0) { + return res.status(404).send('Invalid zoom'); + } - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; + const transformer = isRaw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; - if (transformer) { - const ll = transformer([x, y]); - x = ll[0]; - y = ll[1]; - } + if (transformer) { + const ll = transformer([x, y]); + x = ll[0]; + y = ll[1]; + } - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery(req.query, options, transformer); - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); - // prettier-ignore - return await respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } else if (staticTypeMatch.groups.minx) { - // Area Based Static Image - const bbox = [ - +staticTypeMatch.groups.minx, - +staticTypeMatch.groups.miny, - +staticTypeMatch.groups.maxx, - +staticTypeMatch.groups.maxx, - ]; - let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; - - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - if (transformer) { - const minCorner = transformer(bbox.slice(0, 2)); - const maxCorner = transformer(bbox.slice(2)); - bbox[0] = minCorner[0]; - bbox[1] = minCorner[1]; - bbox[2] = maxCorner[0]; - bbox[3] = maxCorner[1]; - center = transformer(center); - } + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.minx) { + // Area Based Static Image + const bbox = [ + +staticTypeMatch.groups.minx, + +staticTypeMatch.groups.miny, + +staticTypeMatch.groups.maxx, + +staticTypeMatch.groups.maxx, + ]; + let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; + + const transformer = isRaw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + if (transformer) { + const minCorner = transformer(bbox.slice(0, 2)); + const maxCorner = transformer(bbox.slice(2)); + bbox[0] = minCorner[0]; + bbox[1] = minCorner[1]; + bbox[2] = maxCorner[0]; + bbox[3] = maxCorner[1]; + center = transformer(center); + } - const z = calcZForBBox(bbox, w, h, req.query); - const x = center[0]; - const y = center[1]; - const bearing = 0; - const pitch = 0; + const z = calcZForBBox(bbox, parsedWidth, parsedHeight, req.query); + const x = center[0]; + const y = center[1]; + const bearing = 0; + const pitch = 0; + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery(req.query, options, transformer); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.auto) { + // Area Static Image + const bearing = 0; + const pitch = 0; + + const transformer = isRaw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery(req.query, options, transformer); + + // Extract coordinates from markers + const markerCoordinates = []; + for (const marker of markers) { + markerCoordinates.push(marker.location); + } - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); + // Create array with coordinates from markers and path + const coords = [].concat(paths.flat()).concat(markerCoordinates); - // prettier-ignore - return await respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } else if (staticTypeMatch.groups.auto) { - // Area Static Image - const bearing = 0; - const pitch = 0; - - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + // Check if we have at least one coordinate to calculate a bounding box + if (coords.length < 1) { + return res.status(400).send('No coordinates provided'); + } - // Extract coordinates from markers - const markerCoordinates = []; - for (const marker of markers) { - markerCoordinates.push(marker.location); - } + const bbox = [Infinity, Infinity, -Infinity, -Infinity]; + for (const pair of coords) { + bbox[0] = Math.min(bbox[0], pair[0]); + bbox[1] = Math.min(bbox[1], pair[1]); + bbox[2] = Math.max(bbox[2], pair[0]); + bbox[3] = Math.max(bbox[3], pair[1]); + } - // Create array with coordinates from markers and path - const coords = [].concat(paths.flat()).concat(markerCoordinates); + const bbox_ = mercator.convert(bbox, '900913'); + const center = mercator.inverse([ + (bbox_[0] + bbox_[2]) / 2, + (bbox_[1] + bbox_[3]) / 2, + ]); + + // Calculate zoom level + const maxZoom = parseFloat(req.query.maxzoom); + let z = calcZForBBox(bbox, parsedWidth, parsedHeight, req.query); + if (maxZoom > 0) { + z = Math.min(z, maxZoom); + } - // Check if we have at least one coordinate to calculate a bounding box - if (coords.length < 1) { - return res.status(400).send('No coordinates provided'); - } + const x = center[0]; + const y = center[1]; - const bbox = [Infinity, Infinity, -Infinity, -Infinity]; - for (const pair of coords) { - bbox[0] = Math.min(bbox[0], pair[0]); - bbox[1] = Math.min(bbox[1], pair[1]); - bbox[2] = Math.max(bbox[2], pair[0]); - bbox[3] = Math.max(bbox[3], pair[1]); - } + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); - const bbox_ = mercator.convert(bbox, '900913'); - const center = mercator.inverse([ - (bbox_[0] + bbox_[2]) / 2, - (bbox_[1] + bbox_[3]) / 2, - ]); - - // Calculate zoom level - const maxZoom = parseFloat(req.query.maxzoom); - let z = calcZForBBox(bbox, w, h, req.query); - if (maxZoom > 0) { - z = Math.min(z, maxZoom); - } + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); + } else { + return res.sendStatus(404); + } +} - const x = center[0]; - const y = center[1]; +const existingFonts = {}; +let maxScaleFactor = 2; - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); +export const serve_rendered = { + /** + * Initializes the serve_rendered module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {Promise} A promise that resolves to the Express app. + */ + init: async function (options, repo) { + maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); + const app = express().disable('x-powered-by'); - // prettier-ignore - return await respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } else { - return res.sendStatus(404); + app.get( + `/:id{/:p1}/:p2/:p3/:p4{x:p5}{@:scale}{.:format}`, + async (req, res, next) => { + try { + const { p2 } = req.params; + if (p2 === 'static') { + // Route to static if p2 is static + if (options.serveStaticMaps !== false) { + return handleStaticRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, + ); } - } catch (e) { - next('route'); + return res.sendStatus(404); } - }, - ); - } + + return handleTileRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, + ); + } catch (e) { + console.log(e); + return next(e); + } + }, + ); app.get('{/:tileSize}/:id.json', (req, res, next) => { const item = repo[req.params.id]; From 8dc380944b9382d6e9fef9676269fc6050f6d960 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 22:17:46 -0500 Subject: [PATCH 012/104] add allowedTileSizes and option Co-Authored-By: Andrew Calcutt --- src/server.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/server.js b/src/server.js index 3da2ffb04..f0e336ab1 100644 --- a/src/server.js +++ b/src/server.js @@ -16,7 +16,12 @@ import morgan from 'morgan'; import { serve_data } from './serve_data.js'; import { serve_style } from './serve_style.js'; import { serve_font } from './serve_font.js'; -import { getTileUrls, getPublicUrl, isValidHttpUrl } from './utils.js'; +import { + getTileUrls, + getPublicUrl, + isValidHttpUrl, + allowedOptions, +} from './utils.js'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); @@ -99,6 +104,10 @@ async function start(opts) { ? path.resolve(paths.root, paths.files) : path.resolve(__dirname, '../public/files'); + const allowedTileSizes = allowedOptions(['256', '512'], { + defaultValue: options.tileSize || 256, + }); + const startupPromises = []; for (const type of Object.keys(paths)) { @@ -389,17 +398,19 @@ async function start(opts) { } app.get('{/:tileSize}/rendered.json', (req, res, next) => { - const tileSize = parseInt(req.params.tileSize, 10) || undefined; - res.send(addTileJSONs([], req, 'rendered', tileSize)); + const tileSize = allowedTileSizes(req.params['tileSize']); + res.send(addTileJSONs([], req, 'rendered', parseInt(tileSize, 10))); }); + app.get('/data.json', (req, res) => { res.send(addTileJSONs([], req, 'data', undefined)); }); + app.get('{/:tileSize}/index.json', (req, res, next) => { - const tileSize = parseInt(req.params.tileSize, 10) || undefined; + const tileSize = allowedTileSizes(req.params['tileSize']); res.send( addTileJSONs( - addTileJSONs([], req, 'rendered', tileSize), + addTileJSONs([], req, 'rendered', parseInt(tileSize, 10)), req, 'data', undefined, From ab20e81751404c9ec2b1ca06723f89550fff3099 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 22:18:02 -0500 Subject: [PATCH 013/104] cleanup error responses Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 69 +++++++++++++++++++++++----------------------- src/utils.js | 16 +++++++++-- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index d0c2e3b82..fbdda7c66 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -19,7 +19,7 @@ const allowedSpriteFormats = allowedOptions(['png', 'json']); */ function allowedSpriteScales(scale) { if (!scale) return ''; - const match = scale.match(/(\d+)x/); + const match = scale.match(/(\d+)x/); const parsedScale = match ? parseInt(match[1], 10) : 1; return '@' + Math.min(parsedScale, 3) + 'x'; } @@ -64,42 +64,43 @@ export const serve_style = { }); app.get(`/:id/:sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { - console.log(req.params); const { spriteID = 'default', id, format } = req.params; const scale = allowedSpriteScales(req.params.scale); - try { - if ( - !allowedSpriteFormats(format) || - ((id == 256 || id == 512) && format === 'json') - ) { - //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml - next('route'); - } else { - const item = repo[id]; - const sprite = item.spritePaths.find( - (sprite) => sprite.id === spriteID, - ); - if (sprite) { - const filename = `${sprite.path + scale}.${format}`; - return fs.readFile(filename, (err, data) => { - if (err) { - console.log('Sprite load error:', filename); - return res.sendStatus(404); - } else { - if (format === 'json') - res.header('Content-type', 'application/json'); - if (format === 'png') res.header('Content-type', 'image/png'); - return res.send(data); - } - }); - } else { - return res.status(400).send('Bad Sprite ID or Scale'); - } - } - } catch (e) { - console.log(e); - next('route'); + + if ( + !allowedSpriteFormats(format) || + ((id == 256 || id == 512) && format === 'json') + ) { + //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml + return next('route'); } + + const item = repo[id]; + if (!item) { + return res.sendStatus(404); // Ensure item exists first to prevent errors + } + + const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); + if (!sprite) { + return res.status(400).send('Bad Sprite ID or Scale'); + } + + const spriteScale = allowedSpriteScales(scale); + const filename = `${sprite.path}${spriteScale}.${format}`; + + fs.readFile(filename, (err, data) => { + if (err) { + console.error('Sprite load error: %s, Error: %s', filename, err); + return res.sendStatus(404); + } + + if (format === 'json') { + res.header('Content-type', 'application/json'); + } else if (format === 'png') { + res.header('Content-type', 'image/png'); + } + return res.send(data); + }); }); return app; diff --git a/src/utils.js b/src/utils.js index 5d01b5f8f..cd3d17cfb 100644 --- a/src/utils.js +++ b/src/utils.js @@ -184,14 +184,23 @@ export function fixTileJSONCenter(tileJSON) { function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return new Promise((resolve, reject) => { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { + if (!name || typeof name !== 'string' || name.trim() === '') { + console.error('ERROR: Invalid font name: %s', name); + return reject('Invalid font name'); + } + if (!/^\d+-\d+$/.test(range)) { + console.error('ERROR: Invalid range: %s', range); + return reject('Invalid range'); + } const filename = path.join(fontPath, name, `${range}.pbf`); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } delete fallbacks[name]; + // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { - console.error(`ERROR: Font not found: ${name}`); + console.error('ERROR: Font not found: %s, Error: %s', filename, err); if (fallbacks && Object.keys(fallbacks).length) { let fallbackName; @@ -207,7 +216,10 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } } - console.error(`ERROR: Trying to use ${fallbackName} as a fallback`); + console.error( + `ERROR: Trying to use %s as a fallback`, + fallbackName, + ); delete fallbacks[fallbackName]; getFontPbf(null, fontPath, fallbackName, range, fallbacks).then( resolve, From 9f3a7cec4a70220c1931ec7324b6894cce0a14c4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 00:22:14 -0500 Subject: [PATCH 014/104] fix /style/id.json with next('route') Co-Authored-By: Andrew Calcutt --- src/server.js | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/server.js b/src/server.js index f0e336ab1..9b6ca07cd 100644 --- a/src/server.js +++ b/src/server.js @@ -445,33 +445,40 @@ async function start(opts) { try { const content = fs.readFileSync(templateFile, 'utf-8'); const compiled = handlebars.compile(content.toString()); - app.get(urlPath, (req, res) => { - console.log(`Serving template at path: ${urlPath}`); + app.get(urlPath, (req, res, next) => { + if (opts.verbose) { + console.log(`Serving template at path: ${urlPath}`); + } let data = {}; if (dataGetter) { data = dataGetter(req); - if (!data) { - console.error(`Data getter for ${template} returned null`); - return res.status(404).send('Not found'); + if (data) { + data['server_version'] = + `${packageJson.name} v${packageJson.version}`; + data['public_url'] = opts.publicUrl || '/'; + data['is_light'] = isLight; + data['key_query_part'] = req.query.key + ? `key=${encodeURIComponent(req.query.key)}&` + : ''; + data['key_query'] = req.query.key + ? `?key=${encodeURIComponent(req.query.key)}` + : ''; + if (template === 'wmts') res.set('Content-Type', 'text/xml'); + return res.status(200).send(compiled(data)); + } else { + if (opts.verbose) { + console.log(`Forwarding request for: ${urlPath} to next route`); + } + next('route'); } } - data['server_version'] = `${packageJson.name} v${packageJson.version}`; - data['public_url'] = opts.publicUrl || '/'; - data['is_light'] = isLight; - data['key_query_part'] = req.query.key - ? `key=${encodeURIComponent(req.query.key)}&` - : ''; - data['key_query'] = req.query.key - ? `?key=${encodeURIComponent(req.query.key)}` - : ''; - if (template === 'wmts') res.set('Content-Type', 'text/xml'); - return res.status(200).send(compiled(data)); }); } catch (err) { console.error(`Error reading template file: ${templateFile}`, err); throw new Error(`Template not found: ${err.message}`); //throw an error so that the server doesnt start } } + serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { From 1f0ee0dd7b78dd885630f226eb6df4c5885aebd1 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 00:23:33 -0500 Subject: [PATCH 015/104] improve sprite path Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index fbdda7c66..c26eb94a7 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -63,21 +63,13 @@ export const serve_style = { return res.send(styleJSON_); }); - app.get(`/:id/:sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { const { spriteID = 'default', id, format } = req.params; - const scale = allowedSpriteScales(req.params.scale); - - if ( - !allowedSpriteFormats(format) || - ((id == 256 || id == 512) && format === 'json') - ) { - //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml - return next('route'); - } + const spriteScale = allowedSpriteScales(req.params.scale); const item = repo[id]; - if (!item) { - return res.sendStatus(404); // Ensure item exists first to prevent errors + if (!item || !allowedSpriteFormats(format)) { + return res.sendStatus(404); } const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); @@ -85,9 +77,9 @@ export const serve_style = { return res.status(400).send('Bad Sprite ID or Scale'); } - const spriteScale = allowedSpriteScales(scale); const filename = `${sprite.path}${spriteScale}.${format}`; + // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { console.error('Sprite load error: %s, Error: %s', filename, err); From d255cdc14046e7093eaccfbee16b185a713ce415 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 00:24:03 -0500 Subject: [PATCH 016/104] add parseFloadts around zxy Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 6cff9c3f0..d5e61d009 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -33,9 +33,9 @@ export const serve_data = { return res.sendStatus(404); } const tileJSONFormat = item.tileJSON.format; - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; + const z = parseFloat(req.params.z) | 0; + const x = parseFloat(req.params.x) | 0; + const y = parseFloat(req.params.y) | 0; let format = req.params.format; if (format === options.pbfAlias) { format = 'pbf'; From c4adfa84a60b324f43116313301e50cbb52ebdef Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 02:39:47 -0500 Subject: [PATCH 017/104] simplify server_data Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 170 +++++++++++++++++++--------------------------- 1 file changed, 68 insertions(+), 102 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index d5e61d009..0a8bfaecb 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -33,9 +33,13 @@ export const serve_data = { return res.sendStatus(404); } const tileJSONFormat = item.tileJSON.format; - const z = parseFloat(req.params.z) | 0; - const x = parseFloat(req.params.x) | 0; - const y = parseFloat(req.params.y) | 0; + const z = parseInt(req.params.z, 10); + const x = parseInt(req.params.x, 10); + const y = parseInt(req.params.y, 10); + if (isNaN(z) || isNaN(x) || isNaN(y)) { + return res.status(404).send('Invalid Tile'); + } + let format = req.params.format; if (format === options.pbfAlias) { format = 'pbf'; @@ -48,7 +52,6 @@ export const serve_data = { } if ( z < item.tileJSON.minzoom || - 0 || x < 0 || y < 0 || z > item.tileJSON.maxzoom || @@ -57,113 +60,76 @@ export const serve_data = { ) { return res.status(404).send('Out of bounds'); } - if (item.sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(item.source, z, x, y); - if (tileinfo == undefined || tileinfo.data == undefined) { - return res.status(404).send('Not found'); - } else { - let data = tileinfo.data; - let headers = tileinfo.header; - if (tileJSONFormat === 'pbf') { - if (options.dataDecoratorFunc) { - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } - } - data = JSON.stringify(geojson); - } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); - data = await gzipP(data); - - return res.status(200).send(data); - } + let getTile; + if (item.sourceType === 'pmtiles') { + const tileinfo = await getPMtilesTile(item.source, z, x, y); + if (!tileinfo?.data) return res.status(204).send(); + getTile = { data: tileinfo.data, header: tileinfo.header }; } else if (item.sourceType === 'mbtiles') { - item.source.getTile(z, x, y, async (err, data, headers) => { - let isGzipped; - if (err) { - if (/does not exist/.test(err.message)) { - return res.status(204).send(); - } else { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } - } else { - if (data == null) { - return res.status(404).send('Not found'); - } else { - if (tileJSONFormat === 'pbf') { - isGzipped = - data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - if (options.dataDecoratorFunc) { - if (isGzipped) { - data = await gunzipP(data); - isGzipped = false; - } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } + try { + getTile = await new Promise((resolve, reject) => { + item.source.getTile(z, x, y, (err, tileData, tileHeader) => { + if (err) { + return /does not exist/.test(err.message) + ? resolve(null) + : reject(err); } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - - if (isGzipped) { - data = await gunzipP(data); - isGzipped = false; - } + resolve({ data: tileData, header: tileHeader }); + }); + }); + } catch (e) { + return res.status(500).send(e.message); + } + } + if (getTile == null) return res.status(204).send(); - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } - } - data = JSON.stringify(geojson); - } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); + let data = getTile.data; + let headers = getTile.header; + let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - if (!isGzipped) { - data = await gzipP(data); - } + if (tileJSONFormat === 'pbf') { + if (options.dataDecoratorFunc) { + if (isGzipped) { + data = await gunzipP(data); + isGzipped = false; + } + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + } + } - return res.status(200).send(data); - } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); } - }); + } + data = JSON.stringify(geojson); + } + console.log(headers); + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); + + if (!isGzipped) { + data = await gzipP(data); } + + return res.status(200).send(data); }); + app.get('/:id.json', (req, res) => { const item = repo[req.params.id]; if (!item) { From 3aaab828cb51c5cdd9b61caf5631d38c24d9a7fe Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 11:12:20 -0500 Subject: [PATCH 018/104] move tile fetch and add fix verbose logging Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 47 ++++++-------- src/serve_rendered.js | 140 ++++++++++++++++-------------------------- src/serve_style.js | 5 +- src/server.js | 8 +-- src/utils.js | 29 +++++++++ 5 files changed, 108 insertions(+), 121 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 0a8bfaecb..81e257422 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -8,7 +8,12 @@ import express from 'express'; import Pbf from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; -import { fixTileJSONCenter, getTileUrls, isValidHttpUrl } from './utils.js'; +import { + fixTileJSONCenter, + getTileUrls, + isValidHttpUrl, + fetchTileData, +} from './utils.js'; import { getPMtilesInfo, getPMtilesTile, @@ -61,31 +66,17 @@ export const serve_data = { return res.status(404).send('Out of bounds'); } - let getTile; - if (item.sourceType === 'pmtiles') { - const tileinfo = await getPMtilesTile(item.source, z, x, y); - if (!tileinfo?.data) return res.status(204).send(); - getTile = { data: tileinfo.data, header: tileinfo.header }; - } else if (item.sourceType === 'mbtiles') { - try { - getTile = await new Promise((resolve, reject) => { - item.source.getTile(z, x, y, (err, tileData, tileHeader) => { - if (err) { - return /does not exist/.test(err.message) - ? resolve(null) - : reject(err); - } - resolve({ data: tileData, header: tileHeader }); - }); - }); - } catch (e) { - return res.status(500).send(e.message); - } - } - if (getTile == null) return res.status(204).send(); + const fetchTile = await fetchTileData( + item.source, + item.sourceType, + z, + x, + y, + ); + if (fetchTile == null) return res.status(204).send(); - let data = getTile.data; - let headers = getTile.header; + let data = fetchTile.data; + let headers = fetchTile.headers; let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; if (tileJSONFormat === 'pbf') { @@ -118,7 +109,6 @@ export const serve_data = { } data = JSON.stringify(geojson); } - console.log(headers); delete headers['ETag']; // do not trust the tile ETag -- regenerate headers['Content-Encoding'] = 'gzip'; res.set(headers); @@ -159,10 +149,11 @@ export const serve_data = { * @param {object} repo Repository object. * @param {object} params Parameters object. * @param {string} id ID of the data source. - * @param {string} publicUrl Public URL of the data. + * @param {object} programOpts - An object containing the program options * @returns {Promise} */ - add: async function (options, repo, params, id, publicUrl) { + add: async function (options, repo, params, id, programOpts) { + const { publicUrl } = programOpts; let inputFile; let inputType; if (params.pmtiles) { diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 39649477b..944b8410b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -33,12 +33,9 @@ import { getTileUrls, isValidHttpUrl, fixTileJSONCenter, + fetchTileData, } from './utils.js'; -import { - openPMtiles, - getPMtilesInfo, - getPMtilesTile, -} from './pmtiles_adapter.js'; +import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; import { renderOverlay, renderWatermark, renderAttribution } from './render.js'; import fsp from 'node:fs/promises'; import { existsP, gunzipP } from './promises.js'; @@ -951,11 +948,11 @@ export const serve_rendered = { * @param {object} repo Repository object. * @param {object} params Parameters object. * @param {string} id ID of the item. - * @param {string} publicUrl Public URL. + * @param {object} programOpts - An object containing the program options * @param {Function} dataResolver Function to resolve data. * @returns {Promise} */ - add: async function (options, repo, params, id, publicUrl, dataResolver) { + add: async function (options, repo, params, id, programOpts, dataResolver) { const map = { renderers: [], renderersStatic: [], @@ -963,6 +960,8 @@ export const serve_rendered = { sourceTypes: {}, }; + const { publicUrl, verbose } = programOpts; + let styleJSON; /** * Creates a pool of renderers. @@ -1023,88 +1022,57 @@ export const serve_rendered = { const y = parts[5].split('.')[0] | 0; const format = parts[5].split('.')[1]; - if (sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(source, z, x, y); - let data = tileinfo.data; - let headers = tileinfo.header; - if (data == undefined) { - if (options.verbose) - console.log('MBTiles error, serving empty', err); - createEmptyResponse( - sourceInfo.format, - sourceInfo.color, - callback, + const fetchTile = await fetchTileData( + source, + sourceType, + z, + x, + y, + ); + if (fetchTile == null) { + if (verbose) { + console.log( + 'fetchTile error on %s, serving empty response', + req.url, ); - return; - } else { - const response = {}; - response.data = data; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } - } - - callback(null, response); } - } else if (sourceType === 'mbtiles') { - source.getTile(z, x, y, async (err, data, headers) => { - if (err) { - if (options.verbose) - console.log('MBTiles error, serving empty', err); - createEmptyResponse( - sourceInfo.format, - sourceInfo.color, - callback, - ); - return; - } - - const response = {}; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - try { - response.data = await gunzipP(data); - } catch (err) { - console.log( - 'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf', - id, - z, - x, - y, - ); - } - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } - } else { - response.data = data; - } - - callback(null, response); - }); + createEmptyResponse( + sourceInfo.format, + sourceInfo.color, + callback, + ); + return; + } + + const response = {}; + response.data = fetchTile.data; + let headers = fetchTile.headers; + let isGzipped = + response.data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === + 0; + + if (headers['Last-Modified']) { + response.modified = new Date(headers['Last-Modified']); } + + if (format === 'pbf') { + if (isGzipped) { + response.data = await gunzipP(response.data); + isGzipped = false; + } + if (options.dataDecoratorFunc) { + response.data = options.dataDecoratorFunc( + sourceId, + 'data', + response.data, + z, + x, + y, + ); + } + } + + callback(null, response); } else if (protocol === 'http' || protocol === 'https') { try { const response = await axios.get(req.url, { diff --git a/src/serve_style.js b/src/serve_style.js index c26eb94a7..5a7478288 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -112,7 +112,7 @@ export const serve_style = { * @param {object} repo Repository object. * @param {object} params Parameters object containing style path * @param {string} id ID of the style. - * @param {string} publicUrl Public URL of the data. + * @param {object} programOpts - An object containing the program options * @param {Function} reportTiles Function for reporting tile sources. * @param {Function} reportFont Function for reporting font usage * @returns {boolean} true if add is succesful @@ -122,10 +122,11 @@ export const serve_style = { repo, params, id, - publicUrl, + programOpts, reportTiles, reportFont, ) { + const { publicUrl } = programOpts; const styleFile = path.resolve(options.paths.styles, params.style); let styleFileData; diff --git a/src/server.js b/src/server.js index 9b6ca07cd..a49fbac56 100644 --- a/src/server.js +++ b/src/server.js @@ -193,7 +193,7 @@ async function start(opts) { serving.styles, item, id, - opts.publicUrl, + opts, (styleSourceId, protocol) => { let dataItemId; for (const id of Object.keys(data)) { @@ -250,7 +250,7 @@ async function start(opts) { serving.rendered, item, id, - opts.publicUrl, + opts, function dataResolver(styleSourceId) { let fileType; let inputFile; @@ -301,9 +301,7 @@ async function start(opts) { ); continue; } - startupPromises.push( - serve_data.add(options, serving.data, item, id, opts.publicUrl), - ); + startupPromises.push(serve_data.add(options, serving.data, item, id, opts)); } if (options.serveAllStyles) { fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => { diff --git a/src/utils.js b/src/utils.js index cd3d17cfb..bf2d15fac 100644 --- a/src/utils.js +++ b/src/utils.js @@ -6,6 +6,7 @@ import fs from 'node:fs'; import clone from 'clone'; import { combine } from '@jsse/pbfont'; import { existsP } from './promises.js'; +import { getPMtilesTile } from './pmtiles_adapter.js'; /** * Restrict user input to an allowed set of options. @@ -310,3 +311,31 @@ export function isValidHttpUrl(string) { return url.protocol === 'http:' || url.protocol === 'https:'; } + +/** + * Fetches tile data from either PMTiles or MBTiles source. + * @param {object} source - The source object, which may contain a mbtiles object, or pmtiles object. + * @param {string} sourceType - The source type, which should be `pmtiles` or `mbtiles` + * @param {number} z - The zoom level. + * @param {number} x - The x coordinate of the tile. + * @param {number} y - The y coordinate of the tile. + * @returns {Promise} - A promise that resolves to an object with data and headers or null if no data is found. + */ +export async function fetchTileData(source, sourceType, z, x, y) { + if (sourceType === 'pmtiles') { + return await new Promise(async (resolve) => { + const tileinfo = await getPMtilesTile(source, z, x, y); + if (!tileinfo?.data) return resolve(null); + resolve({ data: tileinfo.data, headers: tileinfo.header }); + }); + } else if (sourceType === 'mbtiles') { + return await new Promise((resolve) => { + source.getTile(z, x, y, (err, tileData, tileHeader) => { + if (err) { + return resolve(null); + } + resolve({ data: tileData, headers: tileHeader }); + }); + }); + } +} From 848f7c57af190bf8f7b15ea39797acdf48de3c69 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 11:22:30 -0500 Subject: [PATCH 019/104] add Handling request to verbose logging Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 944b8410b..1f298b1a3 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -984,7 +984,9 @@ export const serve_rendered = { ratio, request: async (req, callback) => { const protocol = req.url.split(':')[0]; - // console.log('Handling request:', req); + if (verbose) { + console.log('Handling request:', req); + } if (protocol === 'sprites') { const dir = options.paths[protocol]; const file = decodeURIComponent(req.url).substring( From 5a883d9db0ebbf7c8d3a562cf74d56feafd4fd42 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 01:06:29 -0500 Subject: [PATCH 020/104] first attempt to upgrade express to v5 Co-Authored-By: Andrew Calcutt --- package-lock.json | 552 +++++++++++++++++++++++++++++------------- package.json | 2 +- src/serve_data.js | 220 +++++++---------- src/serve_font.js | 48 ++-- src/serve_rendered.js | 499 ++++++++++++++++++-------------------- src/serve_style.js | 37 ++- src/server.js | 94 ++++--- test/setup.js | 4 +- 8 files changed, 798 insertions(+), 658 deletions(-) diff --git a/package-lock.json b/package-lock.json index 408b6523e..d8d6a3dd3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "color": "4.2.3", "commander": "12.1.0", "cors": "2.8.5", - "express": "4.19.2", + "express": "5.0.1", "handlebars": "4.7.8", "http-shutdown": "1.2.2", "morgan": "1.10.0", @@ -1721,17 +1721,44 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.0.tgz", + "integrity": "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==", + "dependencies": { + "mime-db": "^1.53.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", @@ -1924,9 +1951,9 @@ } }, "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz", + "integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA==" }, "node_modules/array-ify": { "version": "1.0.0", @@ -2041,41 +2068,58 @@ } }, "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.0.2.tgz", + "integrity": "sha512-SNMk0OONlQ01uk8EPeiBvTW7W4ovpL5b1O3t1sjpPgfxOQ6BqQJ6XjxinDPR79Z6HdcD5zBBwr5ssiTlgdNztQ==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", + "debug": "3.1.0", "destroy": "1.2.0", "http-errors": "2.0.0", - "iconv-lite": "0.4.24", + "iconv-lite": "0.5.2", "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" + "qs": "6.13.0", + "raw-body": "^3.0.0", + "type-is": "~1.6.18" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": ">=18" } }, "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "dependencies": { "ms": "2.0.0" } }, + "node_modules/body-parser/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/body-parser/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/body-parser/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -2156,12 +2200,19 @@ } }, "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "license": "MIT", "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -2492,9 +2543,9 @@ "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==" }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", + "integrity": "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==", "dependencies": { "safe-buffer": "5.2.1" }, @@ -2565,17 +2616,21 @@ } }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/cookiejar": { "version": "2.1.4", @@ -2742,6 +2797,23 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/define-properties": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", @@ -2867,9 +2939,9 @@ "dev": true }, "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "engines": { "node": ">= 0.8" } @@ -2976,6 +3048,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz", @@ -3445,58 +3538,65 @@ } }, "node_modules/express": { - "version": "4.19.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", - "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.2", - "content-disposition": "0.5.4", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.0.1.tgz", + "integrity": "sha512-ORF7g6qGnD+YtUG9yx4DFoqCShNMmUKiXuT5oWMHiOvt/4WFbHC6yCwQMTSBMno7AqntNCAzzcnnjowRkTL9eQ==", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.0.1", + "content-disposition": "^1.0.0", "content-type": "~1.0.4", - "cookie": "0.6.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", + "cookie": "0.7.1", + "cookie-signature": "^1.2.1", + "debug": "4.3.6", "depd": "2.0.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", - "fresh": "0.5.2", + "finalhandler": "^2.0.0", + "fresh": "2.0.0", "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", + "merge-descriptors": "^2.0.0", "methods": "~1.1.2", + "mime-types": "^3.0.0", "on-finished": "2.4.1", + "once": "1.4.0", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", + "router": "^2.0.0", "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", + "send": "^1.1.0", + "serve-static": "^2.1.0", "setprototypeof": "1.2.0", "statuses": "2.0.1", - "type-is": "~1.6.18", + "type-is": "^2.0.0", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "engines": { - "node": ">= 0.10.0" + "node": ">= 18" } }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" + "node_modules/express/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "engines": { + "node": ">= 0.6" } }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "node_modules/express/node_modules/mime-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.0.tgz", + "integrity": "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==", + "dependencies": { + "mime-db": "^1.53.0" + }, + "engines": { + "node": ">= 0.6" + } }, "node_modules/extend-shallow": { "version": "2.0.1", @@ -3599,9 +3699,9 @@ } }, "node_modules/finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.0.0.tgz", + "integrity": "sha512-MX6Zo2adDViYh+GcxxB1dpO43eypOGUOL12rLCOTMQv/DfIbpSJUy4oQIIZhVZkH9e+bZWKMon0XHFEju16tkQ==", "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", @@ -3623,6 +3723,14 @@ "ms": "2.0.0" } }, + "node_modules/finalhandler/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/finalhandler/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -3735,11 +3843,11 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/fs-minipass": { @@ -3772,9 +3880,13 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/function.prototype.name": { "version": "1.1.5", @@ -3877,13 +3989,19 @@ } }, "node_modules/get-intrinsic": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", - "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "license": "MIT", "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4146,11 +4264,12 @@ } }, "node_modules/has-property-descriptors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", - "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.1.1" + "es-define-property": "^1.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4197,6 +4316,18 @@ "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -4295,9 +4426,9 @@ } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz", + "integrity": "sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -4632,6 +4763,11 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==" + }, "node_modules/is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -5305,11 +5441,11 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/memorystream": { @@ -5382,9 +5518,15 @@ } }, "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/merge-stream": { "version": "2.0.0", @@ -5427,17 +5569,6 @@ "node": ">=8.6" } }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -5908,6 +6039,7 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "optional": true, "engines": { "node": ">= 0.6" } @@ -6318,9 +6450,13 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", + "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6543,9 +6679,12 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", + "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "engines": { + "node": ">=16" + } }, "node_modules/path-type": { "version": "4.0.0", @@ -6712,11 +6851,12 @@ } }, "node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -6776,19 +6916,30 @@ } }, "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz", + "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", - "iconv-lite": "0.4.24", + "iconv-lite": "0.6.3", "unpipe": "1.0.0" }, "engines": { "node": ">= 0.8" } }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", @@ -7086,6 +7237,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/router": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.0.0.tgz", + "integrity": "sha512-dIM5zVoG8xhC6rnSN8uoAgFARwTE7BQs8YwHEvK0VCmfxQXMaOuA1uiR1IPwsW7JyK5iTt7Od/TC9StasS2NPQ==", + "dependencies": { + "array-flatten": "3.0.0", + "is-promise": "4.0.0", + "methods": "~1.1.2", + "parseurl": "~1.3.3", + "path-to-regexp": "^8.0.0", + "setprototypeof": "1.2.0", + "utils-merge": "1.0.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -7186,41 +7354,35 @@ } }, "node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.1.0.tgz", + "integrity": "sha512-v67WcEouB5GxbTWL/4NeToqcZiAWEq90N888fczVArY8A79J0L4FD7vj5hm3eUMua5EpoQ59wa/oovY6TLvRUA==", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "debug": "^4.3.5", + "destroy": "^1.2.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^0.5.2", + "http-errors": "^2.0.0", + "mime-types": "^2.1.35", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" } }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" + "node_modules/send/node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" } }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, "node_modules/send/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -7236,17 +7398,17 @@ } }, "node_modules/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.1.0.tgz", + "integrity": "sha512-A3We5UfEjG8Z7VkDv6uItWw6HY2bBSBJT1KtVESn6EOoOr2jAxNhxWCLY3jDE2WcuHXByWju74ck3ZgLwL8xmA==", "dependencies": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.18.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.0.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" } }, "node_modules/set-blocking": { @@ -7254,6 +7416,23 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/set-value": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", @@ -7406,13 +7585,18 @@ "dev": true }, "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "license": "MIT", "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -8110,12 +8294,32 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.0.tgz", + "integrity": "sha512-gd0sGezQYCbWSbkZr75mln4YBidWUN60+devscpLF5mtRDUpiaTvKpBNrdaCvel1NdR2k6vclXybU5fBd2i+nw==", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.0.tgz", + "integrity": "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==", + "dependencies": { + "mime-db": "^1.53.0" }, "engines": { "node": ">= 0.6" diff --git a/package.json b/package.json index 14f94ae0c..0c8dcf320 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "color": "4.2.3", "commander": "12.1.0", "cors": "2.8.5", - "express": "4.19.2", + "express": "5.0.1", "handlebars": "4.7.8", "http-shutdown": "1.2.2", "morgan": "1.10.0", diff --git a/src/serve_data.js b/src/serve_data.js index 1936da6b2..fd45328e4 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -24,149 +24,109 @@ export const serve_data = { init: (options, repo) => { const app = express().disable('x-powered-by'); - app.get( - '/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)', - async (req, res, next) => { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const tileJSONFormat = item.tileJSON.format; - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - let format = req.params.format; - if (format === options.pbfAlias) { - format = 'pbf'; - } - if ( - format !== tileJSONFormat && - !(format === 'geojson' && tileJSONFormat === 'pbf') - ) { - return res.status(404).send('Invalid format'); - } - if ( - z < item.tileJSON.minzoom || - 0 || - x < 0 || - y < 0 || - z > item.tileJSON.maxzoom || - x >= Math.pow(2, z) || - y >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } - if (item.sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(item.source, z, x, y); - if (tileinfo == undefined || tileinfo.data == undefined) { - return res.status(404).send('Not found'); - } else { - let data = tileinfo.data; - let headers = tileinfo.header; - if (tileJSONFormat === 'pbf') { - if (options.dataDecoratorFunc) { - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } + app.get('/:id/:z/:x/:y.:format', async (req, res) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + const tileJSONFormat = item.tileJSON.format; + const z = req.params.z | 0; + const x = req.params.x | 0; + const y = req.params.y | 0; + let format = req.params.format; + if (format === options.pbfAlias) { + format = 'pbf'; + } + if ( + format !== tileJSONFormat && + !(format === 'geojson' && tileJSONFormat === 'pbf') + ) { + return res.status(404).send('Invalid format'); + } + if ( + z < item.tileJSON.minzoom || + 0 || + x < 0 || + y < 0 || + z > item.tileJSON.maxzoom || + x >= Math.pow(2, z) || + y >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } + if (item.sourceType === 'pmtiles') { + let tileinfo = await getPMtilesTile(item.source, z, x, y); + if (tileinfo == undefined || tileinfo.data == undefined) { + return res.status(404).send('Not found'); + } else { + let data = tileinfo.data; + let headers = tileinfo.header; + if (tileJSONFormat === 'pbf') { + if (options.dataDecoratorFunc) { + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } + } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); } - data = JSON.stringify(geojson); } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); + data = JSON.stringify(geojson); + } + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); - data = await gzipP(data); + data = await gzipP(data); - return res.status(200).send(data); - } - } else if (item.sourceType === 'mbtiles') { - item.source.getTile(z, x, y, async (err, data, headers) => { - let isGzipped; - if (err) { - if (/does not exist/.test(err.message)) { - return res.status(204).send(); - } else { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } + return res.status(200).send(data); + } + } else if (item.sourceType === 'mbtiles') { + item.source.getTile(z, x, y, async (err, data, headers) => { + let isGzipped; + if (err) { + if (/does not exist/.test(err.message)) { + return res.status(204).send(); } else { - if (data == null) { - return res.status(404).send('Not found'); - } else { - if (tileJSONFormat === 'pbf') { - isGzipped = - data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - if (options.dataDecoratorFunc) { - if (isGzipped) { - data = await gunzipP(data); - isGzipped = false; - } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - + return res + .status(500) + .header('Content-Type', 'text/plain') + .send(err.message); + } + } else { + if (data == null) { + return res.status(404).send('Not found'); + } else { + if (tileJSONFormat === 'pbf') { + isGzipped = + data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; + if (options.dataDecoratorFunc) { if (isGzipped) { data = await gunzipP(data); isGzipped = false; } - - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } - } - data = JSON.stringify(geojson); - } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); - - if (!isGzipped) { - data = await gzipP(data); + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); } - - return res.status(200).send(data); } - } - }); - } - }, - ); + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; - app.get( - '^/:id/elevation/:z([0-9]+)/:x([-.0-9]+)/:y([-.0-9]+)', + app.get('/:id/elevation/:z/:x/:y', async (req, res, next) => { try { const item = repo?.[req.params.id]; diff --git a/src/serve_font.js b/src/serve_font.js index 02f46dc05..d75390555 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -13,31 +13,29 @@ export const serve_font = async (options, allowedFonts) => { const existingFonts = {}; - app.get( - '/fonts/:fontstack/:range([\\d]+-[\\d]+).pbf', - async (req, res, next) => { - const fontstack = decodeURI(req.params.fontstack); - const range = req.params.range; - - try { - const concatenated = await getFontsPbf( - options.serveAllFonts ? null : allowedFonts, - fontPath, - fontstack, - range, - existingFonts, - ); - - res.header('Content-type', 'application/x-protobuf'); - res.header('Last-Modified', lastModified); - return res.send(concatenated); - } catch (err) { - res.status(400).header('Content-Type', 'text/plain').send(err); - } - }, - ); - - app.get('/fonts.json', (req, res, next) => { + app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + const fontstack = decodeURI(req.params.fontstack); + const range = req.params.range; + + try { + const concatenated = await getFontsPbf( + options.serveAllFonts ? null : allowedFonts, + fontPath, + fontstack, + range, + existingFonts, + ); + + res.header('Content-type', 'application/x-protobuf'); + res.header('Last-Modified', lastModified); + return res.send(concatenated); + } catch (err) { + console.error('Error serving font:', err); + return res.status(400).header('Content-Type', 'text/plain').send(err); + } + }); + + app.get('/fonts.json', (req, res) => { res.header('Content-type', 'application/json'); return res.send( Object.keys(options.serveAllFonts ? existingFonts : allowedFonts).sort(), diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 3e5c94eaa..65246f1bc 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -44,13 +44,43 @@ import fsp from 'node:fs/promises'; import { existsP, gunzipP } from './promises.js'; import { openMbTilesWrapper } from './mbtiles_wrapper.js'; -const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+.?\\d+)'; +const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d*\\.\\d+)'; + +const staticTypeRegex = new RegExp( + `^` + + `(?:` + + // Format 1: {lon},{lat},{zoom}[@{bearing}[,{pitch}]] + `(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN})` + + `(?:@(?${FLOAT_PATTERN})(?:,(?${FLOAT_PATTERN}))?)?` + + `|` + + // Format 2: {minx},{miny},{maxx},{maxy} + `(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN}),(?${FLOAT_PATTERN})` + + `|` + + // Format 3: auto + `(?auto)` + + `)` + + `$`, +); + const PATH_PATTERN = /^((fill|stroke|width)\:[^\|]+\|)*(enc:.+|-?\d+(\.\d*)?,-?\d+(\.\d*)?(\|-?\d+(\.\d*)?,-?\d+(\.\d*)?)+)/; const httpTester = /^https?:\/\//i; const mercator = new SphericalMercator(); -const getScale = (scale) => (scale || '@1x').slice(1, 2) | 0; + +const parseScale = (scale, maxScaleDigit = 9) => { + if (scale === undefined) { + return 1; + } + + // eslint-disable-next-line security/detect-non-literal-regexp + const regex = new RegExp(`^[2-${maxScaleDigit}]x$`); + if (!regex.test(scale)) { + return null; + } + + return parseInt(scale.slice(0, -1), 10); +}; mlgl.on('message', (e) => { if (e.severity === 'WARNING' || e.severity === 'ERROR') { @@ -555,307 +585,256 @@ let maxScaleFactor = 2; export const serve_rendered = { init: async (options, repo) => { maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); - let scalePattern = ''; - for (let i = 2; i <= maxScaleFactor; i++) { - scalePattern += i.toFixed(); - } - scalePattern = `@[${scalePattern}]x`; - const app = express().disable('x-powered-by'); app.get( - `/:id/(:tileSize(256|512)/)?:z(\\d+)/:x(\\d+)/:y(\\d+):scale(${scalePattern})?.:format([\\w]+)`, - (req, res, next) => { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } + `/:id{/:tileSize}/:z/:x/:y{@:scale}{.:format}`, + async (req, res, next) => { + try { + console.log(req.params); + if ( + req.params.z === 'static' || + (req.params.tileSize && + req.params.tileSize != 256 && + req.params.tileSize != 512) + ) { + //workaroud for /:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format} + next('route'); + } else { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(item.lastModified) <= new Date(modifiedSince)) { - return res.sendStatus(304); - } - } + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if (new Date(item.lastModified) <= new Date(modifiedSince)) { + return res.sendStatus(304); + } + } - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - const scale = getScale(req.params.scale); - const format = req.params.format; - const tileSize = parseInt(req.params.tileSize, 10) || 256; - - if ( - z < 0 || - x < 0 || - y < 0 || - z > 22 || - x >= Math.pow(2, z) || - y >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } + const z = req.params.z | 0; + const x = req.params.x | 0; + const y = req.params.y | 0; + const scale = parseScale(req.params.scale, maxScaleFactor); + const format = req.params.format; + const tileSize = parseInt(req.params.tileSize, 10) || 256; + if ( + scale == null || + z < 0 || + x < 0 || + y < 0 || + z > 22 || + x >= Math.pow(2, z) || + y >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } - const tileCenter = mercator.ll( - [ - ((x + 0.5) / (1 << z)) * (256 << z), - ((y + 0.5) / (1 << z)) * (256 << z), - ], - z, - ); + const tileCenter = mercator.ll( + [ + ((x + 0.5) / (1 << z)) * (256 << z), + ((y + 0.5) / (1 << z)) * (256 << z), + ], + z, + ); - // prettier-ignore - return respondImage( - options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, - ); + // prettier-ignore + return await respondImage( + options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, + ); + } + } catch (e) { + console.log(e); + next('route'); + } }, ); if (options.serveStaticMaps !== false) { - const staticPattern = `/:id/static/:raw(raw)?/%s/:width(\\d+)x:height(\\d+):scale(${scalePattern})?.:format([\\w]+)`; - - const centerPattern = util.format( - ':x(%s),:y(%s),:z(%s)(@:bearing(%s)(,:pitch(%s))?)?', - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - ); - app.get( - util.format(staticPattern, centerPattern), + `/:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format}`, async (req, res, next) => { try { const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const raw = req.params.raw; - const z = +req.params.z; - let x = +req.params.x; - let y = +req.params.y; - const bearing = +(req.params.bearing || '0'); - const pitch = +(req.params.pitch || '0'); - const w = req.params.width | 0; - const h = req.params.height | 0; - const scale = getScale(req.params.scale); + console.log(req.params); const format = req.params.format; - - if (z < 0) { - return res.status(404).send('Invalid zoom'); + const w = parseInt(req.params.width) || 512; + const h = parseInt(req.params.height) || 512; + const scale = parseScale(req.params.scale, maxScaleFactor); + let raw = req.params.raw !== undefined; + let type = req.params.type; + if (!type) { + //workaround for type when raw is not set + type = req.params.raw; + raw = false; } - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - if (transformer) { - const ll = transformer([x, y]); - x = ll[0]; - y = ll[1]; + if (!item || !type || !format || !scale) { + return res.sendStatus(404); } - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); - - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); - - // prettier-ignore - return respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } catch (e) { - next(e); - } - }, - ); - - const serveBounds = async (req, res, next) => { - try { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const raw = req.params.raw; - const bbox = [ - +req.params.minx, - +req.params.miny, - +req.params.maxx, - +req.params.maxy, - ]; - let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; - - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - if (transformer) { - const minCorner = transformer(bbox.slice(0, 2)); - const maxCorner = transformer(bbox.slice(2)); - bbox[0] = minCorner[0]; - bbox[1] = minCorner[1]; - bbox[2] = maxCorner[0]; - bbox[3] = maxCorner[1]; - center = transformer(center); - } + const staticTypeMatch = type.match(staticTypeRegex); + console.log(staticTypeMatch.groups); + if (staticTypeMatch.groups.lon) { + // Center Based Static Image + const z = staticTypeMatch.groups.zoom; + let x = staticTypeMatch.groups.lon; + let y = staticTypeMatch.groups.lat; + const bearing = staticTypeMatch.groups.bearing; + const pitch = staticTypeMatch.groups.pitch; + + if (z < 0) { + return res.status(404).send('Invalid zoom'); + } - const w = req.params.width | 0; - const h = req.params.height | 0; - const scale = getScale(req.params.scale); - const format = req.params.format; - - const z = calcZForBBox(bbox, w, h, req.query); - const x = center[0]; - const y = center[1]; - const bearing = 0; - const pitch = 0; - - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + const transformer = raw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); + if (transformer) { + const ll = transformer([x, y]); + x = ll[0]; + y = ll[1]; + } - // prettier-ignore - return respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } catch (e) { - next(e); - } - }; + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery( + req.query, + options, + transformer, + ); - const boundsPattern = util.format( - ':minx(%s),:miny(%s),:maxx(%s),:maxy(%s)', - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - FLOAT_PATTERN, - ); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, + ); - app.get(util.format(staticPattern, boundsPattern), serveBounds); + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.minx) { + // Area Based Static Image + const bbox = [ + +staticTypeMatch.groups.minx, + +staticTypeMatch.groups.miny, + +staticTypeMatch.groups.maxx, + +staticTypeMatch.groups.maxx, + ]; + let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; + + const transformer = raw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + if (transformer) { + const minCorner = transformer(bbox.slice(0, 2)); + const maxCorner = transformer(bbox.slice(2)); + bbox[0] = minCorner[0]; + bbox[1] = minCorner[1]; + bbox[2] = maxCorner[0]; + bbox[3] = maxCorner[1]; + center = transformer(center); + } - app.get('/:id/static/', (req, res, next) => { - for (const key in req.query) { - req.query[key.toLowerCase()] = req.query[key]; - } - req.params.raw = true; - req.params.format = (req.query.format || 'image/png').split('/').pop(); - const bbox = (req.query.bbox || '').split(','); - req.params.minx = bbox[0]; - req.params.miny = bbox[1]; - req.params.maxx = bbox[2]; - req.params.maxy = bbox[3]; - req.params.width = req.query.width || '256'; - req.params.height = req.query.height || '256'; - if (req.query.scale) { - req.params.width /= req.query.scale; - req.params.height /= req.query.scale; - req.params.scale = `@${req.query.scale}`; - } + const z = calcZForBBox(bbox, w, h, req.query); + const x = center[0]; + const y = center[1]; + const bearing = 0; + const pitch = 0; + + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery( + req.query, + options, + transformer, + ); - return serveBounds(req, res, next); - }); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, + ); - const autoPattern = 'auto'; + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.auto) { + // Area Static Image + const bearing = 0; + const pitch = 0; + + const transformer = raw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery( + req.query, + options, + transformer, + ); - app.get( - util.format(staticPattern, autoPattern), - async (req, res, next) => { - try { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } - const raw = req.params.raw; - const w = req.params.width | 0; - const h = req.params.height | 0; - const bearing = 0; - const pitch = 0; - const scale = getScale(req.params.scale); - const format = req.params.format; + // Extract coordinates from markers + const markerCoordinates = []; + for (const marker of markers) { + markerCoordinates.push(marker.location); + } - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; + // Create array with coordinates from markers and path + const coords = [].concat(paths.flat()).concat(markerCoordinates); - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + // Check if we have at least one coordinate to calculate a bounding box + if (coords.length < 1) { + return res.status(400).send('No coordinates provided'); + } - // Extract coordinates from markers - const markerCoordinates = []; - for (const marker of markers) { - markerCoordinates.push(marker.location); - } + const bbox = [Infinity, Infinity, -Infinity, -Infinity]; + for (const pair of coords) { + bbox[0] = Math.min(bbox[0], pair[0]); + bbox[1] = Math.min(bbox[1], pair[1]); + bbox[2] = Math.max(bbox[2], pair[0]); + bbox[3] = Math.max(bbox[3], pair[1]); + } - // Create array with coordinates from markers and path - const coords = [].concat(paths.flat()).concat(markerCoordinates); + const bbox_ = mercator.convert(bbox, '900913'); + const center = mercator.inverse([ + (bbox_[0] + bbox_[2]) / 2, + (bbox_[1] + bbox_[3]) / 2, + ]); + + // Calculate zoom level + const maxZoom = parseFloat(req.query.maxzoom); + let z = calcZForBBox(bbox, w, h, req.query); + if (maxZoom > 0) { + z = Math.min(z, maxZoom); + } - // Check if we have at least one coordinate to calculate a bounding box - if (coords.length < 1) { - return res.status(400).send('No coordinates provided'); - } + const x = center[0]; + const y = center[1]; - const bbox = [Infinity, Infinity, -Infinity, -Infinity]; - for (const pair of coords) { - bbox[0] = Math.min(bbox[0], pair[0]); - bbox[1] = Math.min(bbox[1], pair[1]); - bbox[2] = Math.max(bbox[2], pair[0]); - bbox[3] = Math.max(bbox[3], pair[1]); - } + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, + ); - const bbox_ = mercator.convert(bbox, '900913'); - const center = mercator.inverse([ - (bbox_[0] + bbox_[2]) / 2, - (bbox_[1] + bbox_[3]) / 2, - ]); - - // Calculate zoom level - const maxZoom = parseFloat(req.query.maxzoom); - let z = calcZForBBox(bbox, w, h, req.query); - if (maxZoom > 0) { - z = Math.min(z, maxZoom); + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', + ); + } else { + return res.sendStatus(404); } - - const x = center[0]; - const y = center[1]; - - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); - - // prettier-ignore - return respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); } catch (e) { - next(e); + next('route'); } }, ); } - app.get('/(:tileSize(256|512)/)?:id.json', (req, res, next) => { + app.get('{/:tileSize}/:id.json', (req, res, next) => { const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); diff --git a/src/serve_style.js b/src/serve_style.js index 5d3b4699f..15f925064 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -10,9 +10,15 @@ import { validateStyleMin } from '@maplibre/maplibre-gl-style-spec'; import { fixUrl, allowedOptions } from './utils.js'; const httpTester = /^https?:\/\//i; -const allowedSpriteScales = allowedOptions(['', '@2x', '@3x']); const allowedSpriteFormats = allowedOptions(['png', 'json']); +const allowedSpriteScales = (scale) => { + if (!scale) return ''; // Default to 1 if no scale provided + const match = scale.match(/(\d+)x/); // Match one or more digits before 'x' + const parsedScale = match ? parseInt(match[1], 10) : 1; // Parse the number, or default to 1 if no match + return '@' + Math.min(parsedScale, 3) + 'x'; +}; + export const serve_style = { init: (options, repo) => { const app = express().disable('x-powered-by'); @@ -46,14 +52,18 @@ export const serve_style = { return res.send(styleJSON_); }); - app.get( - '/:id/sprite(/:spriteID)?:scale(@[23]x)?.:format([\\w]+)', - (req, res, next) => { - const { spriteID = 'default', id } = req.params; - const scale = allowedSpriteScales(req.params.scale) || ''; - const format = allowedSpriteFormats(req.params.format); - - if (format) { + app.get(`/:id/:sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + console.log(req.params); + const { spriteID = 'default', id, format } = req.params; + const scale = allowedSpriteScales(req.params.scale); + try { + if ( + !allowedSpriteFormats(format) || + ((id == 256 || id == 512) && format === 'json') + ) { + //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml + next('route'); + } else { const item = repo[id]; const sprite = item.spritePaths.find( (sprite) => sprite.id === spriteID, @@ -74,11 +84,12 @@ export const serve_style = { } else { return res.status(400).send('Bad Sprite ID or Scale'); } - } else { - return res.status(400).send('Bad Sprite Format'); } - }, - ); + } catch (e) { + console.log(e); + next('route'); + } + }); return app; }, diff --git a/src/server.js b/src/server.js index 39808e318..c5ddf3612 100644 --- a/src/server.js +++ b/src/server.js @@ -37,7 +37,7 @@ const serve_rendered = ( * * @param opts */ -function start(opts) { +async function start(opts) { console.log('Starting server'); const app = express().disable('x-powered-by'); @@ -73,7 +73,7 @@ function start(opts) { config = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch (e) { console.log('ERROR: Config file not found or invalid!'); - console.log(' See README.md for instructions and sample data.'); + console.log(' See README.md for instructions and sample data.'); process.exit(1); } } @@ -379,14 +379,14 @@ function start(opts) { return arr; }; - app.get('/(:tileSize(256|512)/)?rendered.json', (req, res, next) => { + app.get('{/:tileSize}/rendered.json', (req, res, next) => { const tileSize = parseInt(req.params.tileSize, 10) || undefined; res.send(addTileJSONs([], req, 'rendered', tileSize)); }); - app.get('/data.json', (req, res, next) => { + app.get('/data.json', (req, res) => { res.send(addTileJSONs([], req, 'data', undefined)); }); - app.get('/(:tileSize(256|512)/)?index.json', (req, res, next) => { + app.get('{/:tileSize}/index.json', (req, res, next) => { const tileSize = parseInt(req.params.tileSize, 10) || undefined; res.send( addTileJSONs( @@ -415,44 +415,38 @@ function start(opts) { templateFile = path.resolve(paths.root, options.frontPage); } } - startupPromises.push( - new Promise((resolve, reject) => { - fs.readFile(templateFile, (err, content) => { - if (err) { - err = new Error(`Template not found: ${err.message}`); - reject(err); - return; + try { + const content = fs.readFileSync(templateFile, 'utf-8'); + const compiled = handlebars.compile(content.toString()); + app.get(urlPath, (req, res) => { + console.log(`Serving template at path: ${urlPath}`); + let data = {}; + if (dataGetter) { + data = dataGetter(req); + if (!data) { + console.error(`Data getter for ${template} returned null`); + return res.status(404).send('Not found'); } - const compiled = handlebars.compile(content.toString()); - - app.use(urlPath, (req, res, next) => { - let data = {}; - if (dataGetter) { - data = dataGetter(req); - if (!data) { - return res.status(404).send('Not found'); - } - } - data['server_version'] = - `${packageJson.name} v${packageJson.version}`; - data['public_url'] = opts.publicUrl || '/'; - data['is_light'] = isLight; - data['key_query_part'] = req.query.key - ? `key=${encodeURIComponent(req.query.key)}&` - : ''; - data['key_query'] = req.query.key - ? `?key=${encodeURIComponent(req.query.key)}` - : ''; - if (template === 'wmts') res.set('Content-Type', 'text/xml'); - return res.status(200).send(compiled(data)); - }); - resolve(); - }); - }), - ); + } + data['server_version'] = `${packageJson.name} v${packageJson.version}`; + data['public_url'] = opts.publicUrl || '/'; + data['is_light'] = isLight; + data['key_query_part'] = req.query.key + ? `key=${encodeURIComponent(req.query.key)}&` + : ''; + data['key_query'] = req.query.key + ? `?key=${encodeURIComponent(req.query.key)}` + : ''; + if (template === 'wmts') res.set('Content-Type', 'text/xml'); + return res.status(200).send(compiled(data)); + }); + } catch (err) { + console.error(`Error reading template file: ${templateFile}`, err); + throw new Error(`Template not found: ${err.message}`); //throw an error so that the server doesnt start + } }; - serveTemplate('/$', 'index', (req) => { + serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { let style = { @@ -552,7 +546,7 @@ function start(opts) { }; }); - serveTemplate('/styles/:id/$', 'viewer', (req) => { + serveTemplate('/styles/:id/', 'viewer', (req) => { const { id } = req.params; const style = clone(((serving.styles || {})[id] || {}).styleJSON); @@ -569,11 +563,6 @@ function start(opts) { }; }); - /* - app.use('/rendered/:id/$', function(req, res, next) { - return res.redirect(301, '/styles/' + req.params.id + '/'); - }); - */ serveTemplate('/styles/:id/wmts.xml', 'wmts', (req) => { const { id } = req.params; const wmts = clone((serving.styles || {})[id]); @@ -605,9 +594,8 @@ function start(opts) { }; }); - serveTemplate('^/data/(:preview(preview)/)?:id/$', 'data', (req) => { - const id = req.params.id; - const preview = req.params.preview || undefined; + serveTemplate('^/data{/:view}/:id/', 'data', (req) => { + const { id, view } = req.params; const data = serving.data[id]; if (!data) { @@ -616,7 +604,7 @@ function start(opts) { const is_terrain = (data.tileJSON.encoding === 'terrarium' || data.tileJSON.encoding === 'mapbox') && - preview === 'preview'; + view === 'preview'; return { ...data, id, @@ -633,7 +621,7 @@ function start(opts) { startupComplete = true; }); - app.get('/health', (req, res, next) => { + app.get('/health', (req, res) => { if (startupComplete) { return res.status(200).send('OK'); } else { @@ -676,8 +664,8 @@ function stopGracefully(signal) { * * @param opts */ -export function server(opts) { - const running = start(opts); +export async function server(opts) { + const running = await start(opts); running.startupPromise.catch((err) => { console.error(err.message); diff --git a/test/setup.js b/test/setup.js index 34fba6707..1852a195c 100644 --- a/test/setup.js +++ b/test/setup.js @@ -7,10 +7,10 @@ import { server } from '../src/server.js'; global.expect = expect; global.supertest = supertest; -before(function () { +before(async function () { console.log('global setup'); process.chdir('test_data'); - const running = server({ + const running = await server({ configPath: 'config.json', port: 8888, publicUrl: '/test/', From c9bc1ec57306a72cded4e583534b38b9c4c094d8 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 01:56:15 -0500 Subject: [PATCH 021/104] try to fix https://github.com/maptiler/tileserver-gl/issues/1411 Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 65246f1bc..05d14ceb8 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -133,23 +133,31 @@ function createEmptyResponse(format, color, callback) { } // create an "empty" response image - color = new Color(color); - const array = color.array(); - const channels = array.length === 4 && format !== 'jpeg' ? 4 : 3; - sharp(Buffer.from(array), { - raw: { - width: 1, - height: 1, - channels, - }, - }) - .toFormat(format) - .toBuffer((err, buffer, info) => { - if (!err) { + try { + color = new Color(color); + const array = color.array(); + const channels = array.length === 4 && format !== 'jpeg' ? 4 : 3; + sharp(Buffer.from(array), { + raw: { + width: 1, + height: 1, + channels, + }, + }) + .toFormat(format) + .toBuffer((err, buffer, info) => { + if (err) { + console.error('Error creating image with Sharp:', err); + callback(err, null); + return; + } cachedEmptyResponses[cacheKey] = buffer; - } - callback(null, { data: buffer }); - }); + callback(null, { data: buffer }); + }); + } catch (error) { + console.error('Error during image processing setup:', error); + callback(error, null); + } } /** From 12b818b7246b6e40203b5aa9eed65cd72de91e25 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 02:28:42 -0500 Subject: [PATCH 022/104] cleanup server.js Co-Authored-By: Andrew Calcutt --- src/server.js | 64 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/src/server.js b/src/server.js index c5ddf3612..c8186f4ad 100644 --- a/src/server.js +++ b/src/server.js @@ -34,8 +34,9 @@ const serve_rendered = ( ).serve_rendered; /** - * - * @param opts + * Starts the server. + * @param {object} opts - Configuration options for the server. + * @returns {Promise} - A promise that resolves to the server object. */ async function start(opts) { console.log('Starting server'); @@ -116,8 +117,9 @@ async function start(opts) { * Recursively get all files within a directory. * Inspired by https://stackoverflow.com/a/45130990/10133863 * @param {string} directory Absolute path to a directory to get files from. + * @returns {Promise} - A promise that resolves to an array of file paths relative to the icon directory. */ - const getFiles = async (directory) => { + async function getFiles(directory) { // Fetch all entries of the directory and attach type information const dirEntries = await fs.promises.readdir(directory, { withFileTypes: true, @@ -136,7 +138,7 @@ async function start(opts) { // Flatten the list of files to a single array return files.flat(); - }; + } // Load all available icons into a settings object startupPromises.push( @@ -169,8 +171,15 @@ async function start(opts) { }), ); } - - const addStyle = (id, item, allowMoreData, reportFonts) => { + /** + * Adds a style to the server. + * @param {string} id - The ID of the style. + * @param {object} item - The style configuration object. + * @param {boolean} allowMoreData - Whether to allow adding more data sources. + * @param {boolean} reportFonts - Whether to report fonts. + * @returns {void} + */ + function addStyle(id, item, allowMoreData, reportFonts) { let success = true; if (item.serve_data !== false) { success = serve_style.add( @@ -261,7 +270,7 @@ async function start(opts) { item.serve_rendered = false; } } - }; + } for (const id of Object.keys(config.styles || {})) { const item = config.styles[id]; @@ -272,13 +281,11 @@ async function start(opts) { addStyle(id, item, true, true); } - startupPromises.push( serve_font(options, serving.fonts).then((sub) => { app.use('/', sub); }), ); - for (const id of Object.keys(data)) { const item = data[id]; const fileType = Object.keys(data[id])[0]; @@ -288,12 +295,10 @@ async function start(opts) { ); continue; } - startupPromises.push( serve_data.add(options, serving.data, item, id, opts.publicUrl), ); } - if (options.serveAllStyles) { fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => { if (err) { @@ -333,7 +338,6 @@ async function start(opts) { } }); } - app.get('/styles.json', (req, res, next) => { const result = []; const query = req.query.key @@ -354,7 +358,15 @@ async function start(opts) { res.send(result); }); - const addTileJSONs = (arr, req, type, tileSize) => { + /** + * Adds TileJSON metadata to an array. + * @param {Array} arr - The array to add TileJSONs to + * @param {object} req - The express request object. + * @param {string} type - The type of resource + * @param {number} tileSize - The tile size. + * @returns {Array} - An array of TileJSON objects. + */ + function addTileJSONs(arr, req, type, tileSize) { for (const id of Object.keys(serving[type])) { const info = clone(serving[type][id].tileJSON); let path = ''; @@ -377,7 +389,7 @@ async function start(opts) { arr.push(info); } return arr; - }; + } app.get('{/:tileSize}/rendered.json', (req, res, next) => { const tileSize = parseInt(req.params.tileSize, 10) || undefined; @@ -403,7 +415,14 @@ async function start(opts) { app.use('/', express.static(path.join(__dirname, '../public/resources'))); const templates = path.join(__dirname, '../public/templates'); - const serveTemplate = (urlPath, template, dataGetter) => { + /** + * Serves a Handlebars template. + * @param {string} urlPath - The URL path to serve the template at + * @param {string} template - The name of the template file + * @param {Function} dataGetter - A function to get data to be passed to the template. + * @returns {void} + */ + function serveTemplate(urlPath, template, dataGetter) { let templateFile = `${templates}/${template}.tmpl`; if (template === 'index') { if (options.frontPage === false) { @@ -444,8 +463,7 @@ async function start(opts) { console.error(`Error reading template file: ${templateFile}`, err); throw new Error(`Template not found: ${err.message}`); //throw an error so that the server doesnt start } - }; - + } serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { @@ -478,7 +496,6 @@ async function start(opts) { styles[id] = style; } - let datas = {}; for (const id of Object.keys(serving.data || {})) { let data = Object.assign({}, serving.data[id]); @@ -536,10 +553,8 @@ async function start(opts) { } data.formatted_filesize = `${size.toFixed(2)} ${suffix}`; } - datas[id] = data; } - return { styles: Object.keys(styles).length ? styles : null, data: Object.keys(datas).length ? datas : null, @@ -553,7 +568,6 @@ async function start(opts) { if (!style) { return null; } - return { ...style, id, @@ -605,6 +619,7 @@ async function start(opts) { (data.tileJSON.encoding === 'terrarium' || data.tileJSON.encoding === 'mapbox') && view === 'preview'; + return { ...data, id, @@ -654,6 +669,7 @@ async function start(opts) { /** * Stop the server gracefully * @param {string} signal Name of the received signal + * @returns {void} */ function stopGracefully(signal) { console.log(`Caught signal ${signal}, stopping gracefully`); @@ -661,8 +677,9 @@ function stopGracefully(signal) { } /** - * - * @param opts + * Starts and manages the server + * @param {object} opts - Configuration options for the server. + * @returns {Promise} - A promise that resolves to the running server */ export async function server(opts) { const running = await start(opts); @@ -685,6 +702,5 @@ export async function server(opts) { running.app = restarted.app; }); }); - return running; } From b6537b58a0565e46d5258da32006dbab7d789745 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 02:34:39 -0500 Subject: [PATCH 023/104] cleanup serve_font.js Co-Authored-By: Andrew Calcutt --- src/serve_font.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index d75390555..30f1fc8dc 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -4,7 +4,13 @@ import express from 'express'; import { getFontsPbf, listFonts } from './utils.js'; -export const serve_font = async (options, allowedFonts) => { +/** + * Initializes and returns an Express app that serves font files. + * @param {object} options - Configuration options for the server. + * @param {object} allowedFonts - An object containing allowed fonts. + * @returns {Promise} - A promise that resolves to the Express app. + */ +export async function serve_font(options, allowedFonts) { const app = express().disable('x-powered-by'); const lastModified = new Date().toUTCString(); @@ -45,4 +51,4 @@ export const serve_font = async (options, allowedFonts) => { const fonts = await listFonts(options.paths.fonts); Object.assign(existingFonts, fonts); return app; -}; +} From d52154605af70b4c7bd20ab526970556edff4dbf Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 02:58:59 -0500 Subject: [PATCH 024/104] cleanup sever_rendered.js Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 123 +++++++++++++++++++++++++++++++++--------- 1 file changed, 98 insertions(+), 25 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 05d14ceb8..f3b5d94e5 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -68,7 +68,13 @@ const httpTester = /^https?:\/\//i; const mercator = new SphericalMercator(); -const parseScale = (scale, maxScaleDigit = 9) => { +/** + * Parses a scale string to a number. + * @param {string} scale The scale string (e.g., '2x', '4x'). + * @param {number} maxScaleDigit Maximum allowed scale digit. + * @returns {number|null} The parsed scale as a number or null if invalid. + */ +function parseScale(scale, maxScaleDigit = 9) { if (scale === undefined) { return 1; } @@ -80,7 +86,7 @@ const parseScale = (scale, maxScaleDigit = 9) => { } return parseInt(scale.slice(0, -1), 10); -}; +} mlgl.on('message', (e) => { if (e.severity === 'WARNING' || e.severity === 'ERROR') { @@ -111,6 +117,7 @@ const cachedEmptyResponses = { * @param {string} format The format (a sharp format or 'pbf'). * @param {string} color The background color (or empty string for transparent). * @param {Function} callback The mlgl callback. + * @returns {void} */ function createEmptyResponse(format, color, callback) { if (!format || format === 'pbf') { @@ -163,11 +170,12 @@ function createEmptyResponse(format, color, callback) { /** * Parses coordinate pair provided to pair of floats and ensures the resulting * pair is a longitude/latitude combination depending on lnglat query parameter. - * @param {List} coordinatePair Coordinate pair. + * @param {Array} coordinatePair Coordinate pair. * @param coordinates * @param {object} query Request query parameters. + * @returns {Array|null} Parsed coordinate pair as [longitude, latitude] or null if invalid */ -const parseCoordinatePair = (coordinates, query) => { +function parseCoordinatePair(coordinates, query) { const firstCoordinate = parseFloat(coordinates[0]); const secondCoordinate = parseFloat(coordinates[1]); @@ -183,15 +191,16 @@ const parseCoordinatePair = (coordinates, query) => { } return [firstCoordinate, secondCoordinate]; -}; +} /** * Parses a coordinate pair from query arguments and optionally transforms it. - * @param {List} coordinatePair Coordinate pair. + * @param {Array} coordinatePair Coordinate pair. * @param {object} query Request query parameters. * @param {Function} transformer Optional transform function. + * @returns {Array|null} Transformed coordinate pair or null if invalid. */ -const parseCoordinates = (coordinatePair, query, transformer) => { +function parseCoordinates(coordinatePair, query, transformer) { const parsedCoordinates = parseCoordinatePair(coordinatePair, query); // Transform coordinates @@ -200,14 +209,15 @@ const parseCoordinates = (coordinatePair, query, transformer) => { } return parsedCoordinates; -}; +} /** * Parses paths provided via query into a list of path objects. * @param {object} query Request query parameters. * @param {Function} transformer Optional transform function. + * @returns {Array>>} Array of paths. */ -const extractPathsFromQuery = (query, transformer) => { +function extractPathsFromQuery(query, transformer) { // Initiate paths array const paths = []; // Return an empty list if no paths have been provided @@ -259,17 +269,18 @@ const extractPathsFromQuery = (query, transformer) => { } } return paths; -}; +} /** * Parses marker options provided via query and sets corresponding attributes * on marker object. * Options adhere to the following format * [optionName]:[optionValue] - * @param {List[String]} optionsList List of option strings. + * @param {Array} optionsList List of option strings. * @param {object} marker Marker object to configure. + * @returns {void} */ -const parseMarkerOptions = (optionsList, marker) => { +function parseMarkerOptions(optionsList, marker) { for (const options of optionsList) { const optionParts = options.split(':'); // Ensure we got an option name and value @@ -296,15 +307,16 @@ const parseMarkerOptions = (optionsList, marker) => { break; } } -}; +} /** * Parses markers provided via query into a list of marker objects. * @param {object} query Request query parameters. * @param {object} options Configuration options. * @param {Function} transformer Optional transform function. + * @returns {Array} An array of marker objects. */ -const extractMarkersFromQuery = (query, options, transformer) => { +function extractMarkersFromQuery(query, options, transformer) { // Return an empty list if no markers have been provided if (!query.marker) { return []; @@ -380,9 +392,16 @@ const extractMarkersFromQuery = (query, options, transformer) => { markers.push(marker); } return markers; -}; - -const calcZForBBox = (bbox, w, h, query) => { +} +/** + * Calculates the zoom level for a given bounding box. + * @param {Array} bbox Bounding box as [minx, miny, maxx, maxy]. + * @param {number} w Width of the image. + * @param {number} h Height of the image. + * @param {object} query Request query parameters. + * @returns {number} Calculated zoom level. + */ +function calcZForBBox(bbox, w, h, query) { let z = 25; const padding = query.padding !== undefined ? parseFloat(query.padding) : 0.1; @@ -401,9 +420,27 @@ const calcZForBBox = (bbox, w, h, query) => { z = Math.max(Math.log(Math.max(w, h) / 256) / Math.LN2, Math.min(25, z)); return z; -}; +} -const respondImage = ( +/** + * Responds with an image. + * @param {object} options Configuration options. + * @param {object} item Item object containing map and other information. + * @param {number} z Zoom level. + * @param {number} lon Longitude of the center. + * @param {number} lat Latitude of the center. + * @param {number} bearing Map bearing. + * @param {number} pitch Map pitch. + * @param {number} width Width of the image. + * @param {number} height Height of the image. + * @param {number} scale Scale factor. + * @param {string} format Image format. + * @param {object} res Express response object. + * @param {Buffer|null} overlay Optional overlay image. + * @param {string} mode Rendering mode ('tile' or 'static'). + * @returns {Promise} + */ +const respondImage = async ( options, item, z, @@ -451,7 +488,7 @@ const respondImage = ( } else { pool = item.map.renderersStatic[scale]; } - pool.acquire((err, renderer) => { + pool.acquire(async (err, renderer) => { // For 512px tiles, use the actual maplibre-native zoom. For 256px tiles, use zoom - 1 let mlglZ; if (width === 512) { @@ -591,7 +628,13 @@ const existingFonts = {}; let maxScaleFactor = 2; export const serve_rendered = { - init: async (options, repo) => { + /** + * Initializes the serve_rendered module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {Promise} A promise that resolves to the Express app. + */ + init: async function (options, repo) { maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); const app = express().disable('x-powered-by'); @@ -650,8 +693,8 @@ export const serve_rendered = { // prettier-ignore return await respondImage( - options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, - ); + options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, + ); } } catch (e) { console.log(e); @@ -864,7 +907,17 @@ export const serve_rendered = { Object.assign(existingFonts, fonts); return app; }, - add: async (options, repo, params, id, publicUrl, dataResolver) => { + /** + * Adds a new item to the repository. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @param {object} params Parameters object. + * @param {string} id ID of the item. + * @param {string} publicUrl Public URL. + * @param {Function} dataResolver Function to resolve data. + * @returns {Promise} + */ + add: async function (options, repo, params, id, publicUrl, dataResolver) { const map = { renderers: [], renderersStatic: [], @@ -873,7 +926,21 @@ export const serve_rendered = { }; let styleJSON; + /** + * Creates a pool of renderers. + * @param {number} ratio Pixel ratio + * @param {string} mode Rendering mode ('tile' or 'static'). + * @param {number} min Minimum pool size. + * @param {number} max Maximum pool size. + * @returns {object} The created pool + */ const createPool = (ratio, mode, min, max) => { + /** + * Creates a renderer + * @param {number} ratio Pixel ratio + * @param {Function} createCallback Function that returns the renderer when created + * @returns {void} + */ const createRenderer = (ratio, createCallback) => { const renderer = new mlgl.Map({ mode, @@ -1278,7 +1345,13 @@ export const serve_rendered = { ); } }, - remove: (repo, id) => { + /** + * Removes an item from the repository. + * @param {object} repo Repository object. + * @param {string} id ID of the item to remove. + * @returns {void} + */ + remove: function (repo, id) { const item = repo[id]; if (item) { item.map.renderers.forEach((pool) => { From 279421c349495fc6abfc30f268dde018b0c220f9 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 03:09:09 -0500 Subject: [PATCH 025/104] cleanup server_data.js Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index fd45328e4..245ef004e 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -21,7 +21,13 @@ import { gunzipP, gzipP } from './promises.js'; import { openMbTilesWrapper } from './mbtiles_wrapper.js'; export const serve_data = { - init: (options, repo) => { + /** + * Initializes the serve_data module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {express.Application} The initialized Express application. + */ + init: function (options, repo) { const app = express().disable('x-powered-by'); app.get('/:id/:z/:x/:y.:format', async (req, res) => { @@ -325,7 +331,16 @@ export const serve_data = { return app; }, - add: async (options, repo, params, id, publicUrl) => { + /** + * Adds a new data source to the repository. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @param {object} params Parameters object. + * @param {string} id ID of the data source. + * @param {string} publicUrl Public URL of the data. + * @returns {Promise} + */ + add: async function (options, repo, params, id, publicUrl) { let inputFile; let inputType; if (params.pmtiles) { From 8ea4b50e8ff70905d4b3fb639f6b53c722e5f0cf Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 03:15:54 -0500 Subject: [PATCH 026/104] cleanup serve_style Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 15f925064..ec8a0b448 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -12,15 +12,26 @@ import { fixUrl, allowedOptions } from './utils.js'; const httpTester = /^https?:\/\//i; const allowedSpriteFormats = allowedOptions(['png', 'json']); -const allowedSpriteScales = (scale) => { +/** + * Checks and formats sprite scale + * @param {string} scale string containing the scale + * @returns {string} formated string for the scale or empty string if scale is invalid + */ +function allowedSpriteScales(scale) { if (!scale) return ''; // Default to 1 if no scale provided const match = scale.match(/(\d+)x/); // Match one or more digits before 'x' const parsedScale = match ? parseInt(match[1], 10) : 1; // Parse the number, or default to 1 if no match return '@' + Math.min(parsedScale, 3) + 'x'; -}; +} export const serve_style = { - init: (options, repo) => { + /** + * Initializes the serve_style module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {express.Application} The initialized Express application. + */ + init: function (options, repo) { const app = express().disable('x-powered-by'); app.get('/:id/style.json', (req, res, next) => { @@ -93,10 +104,35 @@ export const serve_style = { return app; }, - remove: (repo, id) => { + /** + * Removes an item from the repository. + * @param {object} repo Repository object. + * @param {string} id ID of the item to remove. + * @returns {void} + */ + remove: function (repo, id) { delete repo[id]; }, - add: (options, repo, params, id, publicUrl, reportTiles, reportFont) => { + /** + * Adds a new style to the repository. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @param {object} params Parameters object containing style path + * @param {string} id ID of the style. + * @param {string} publicUrl Public URL of the data. + * @param {Function} reportTiles Function for reporting tile sources. + * @param {Function} reportFont Function for reporting font usage + * @returns {boolean} true if add is succesful + */ + add: function ( + options, + repo, + params, + id, + publicUrl, + reportTiles, + reportFont, + ) { const styleFile = path.resolve(options.paths.styles, params.style); let styleFileData; From 7cbafe8839e16109ba647a1ad6deefe71c82a96d Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 03:19:21 -0500 Subject: [PATCH 027/104] Update serve_style.js Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index ec8a0b448..d0c2e3b82 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -18,9 +18,9 @@ const allowedSpriteFormats = allowedOptions(['png', 'json']); * @returns {string} formated string for the scale or empty string if scale is invalid */ function allowedSpriteScales(scale) { - if (!scale) return ''; // Default to 1 if no scale provided - const match = scale.match(/(\d+)x/); // Match one or more digits before 'x' - const parsedScale = match ? parseInt(match[1], 10) : 1; // Parse the number, or default to 1 if no match + if (!scale) return ''; + const match = scale.match(/(\d+)x/); + const parsedScale = match ? parseInt(match[1], 10) : 1; return '@' + Math.min(parsedScale, 3) + 'x'; } From 08e1d345be23b727751cfea072d369dbd5128d36 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 11:39:38 -0500 Subject: [PATCH 028/104] Move UV_THREADPOOL_SIZE to main thred Co-Authored-By: Andrew Calcutt --- src/main.js | 6 ++++++ src/server.js | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main.js b/src/main.js index 7523aa937..b1f14a239 100644 --- a/src/main.js +++ b/src/main.js @@ -1,6 +1,12 @@ #!/usr/bin/env node 'use strict'; +import os from 'os'; + +const envSize = parseInt(process.env.UV_THREADPOOL_SIZE, 10); +process.env.UV_THREADPOOL_SIZE = Math.ceil( + Math.max(4, isNaN(envSize) ? os.cpus().length * 1.5 : envSize), +); import fs from 'node:fs'; import fsp from 'node:fs/promises'; diff --git a/src/server.js b/src/server.js index c8186f4ad..1dbad650d 100644 --- a/src/server.js +++ b/src/server.js @@ -1,9 +1,6 @@ #!/usr/bin/env node 'use strict'; -import os from 'os'; -process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, os.cpus().length * 1.5)); - import fs from 'node:fs'; import path from 'path'; import fnv1a from '@sindresorhus/fnv1a'; From c0e14bd14559da145f574cef9933c1501fda177a Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 17:05:01 -0500 Subject: [PATCH 029/104] cleanup utils.js Co-Authored-By: Andrew Calcutt --- src/utils.js | 110 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 29 deletions(-) diff --git a/src/utils.js b/src/utils.js index 85dad1a1a..158f052f4 100644 --- a/src/utils.js +++ b/src/utils.js @@ -9,9 +9,10 @@ import { existsP } from './promises.js'; /** * Restrict user input to an allowed set of options. - * @param opts - * @param root0 - * @param root0.defaultValue + * @param {string[]} opts - An array of allowed option strings. + * @param {object} [config] - Optional configuration object. + * @param {string} [config.defaultValue] - The default value to return if input doesn't match. + * @returns {function(string): string} - A function that takes a value and returns it if valid or a default. */ export function allowedOptions(opts, { defaultValue } = {}) { const values = Object.fromEntries(opts.map((key) => [key, key])); @@ -19,10 +20,11 @@ export function allowedOptions(opts, { defaultValue } = {}) { } /** - * Replace local:// urls with public http(s):// urls - * @param req - * @param url - * @param publicUrl + * Replaces local:// URLs with public http(s):// URLs. + * @param {object} req - Express request object. + * @param {string} url - The URL string to fix. + * @param {string} publicUrl - The public URL prefix to use for replacements. + * @returns {string} - The fixed URL string. */ export function fixUrl(req, url, publicUrl) { if (!url || typeof url !== 'string' || url.indexOf('local://') !== 0) { @@ -40,12 +42,11 @@ export function fixUrl(req, url, publicUrl) { } /** - * Generate new URL object - * @param req - * @params {object} req - Express request - * @returns {URL} object + * Generates a new URL object from the Express request. + * @param {object} req - Express request object. + * @returns {URL} - URL object with correct host and optionally path. */ -const getUrlObject = (req) => { +function getUrlObject(req) { const urlObject = new URL(`${req.protocol}://${req.headers.host}/`); // support overriding hostname by sending X-Forwarded-Host http header urlObject.hostname = req.hostname; @@ -62,16 +63,33 @@ const getUrlObject = (req) => { urlObject.pathname = path.posix.join(xForwardedPath, urlObject.pathname); } return urlObject; -}; +} -export const getPublicUrl = (publicUrl, req) => { +/** + * Gets the public URL, either from a provided publicUrl or generated from the request. + * @param {string} publicUrl - The optional public URL to use. + * @param {object} req - The Express request object. + * @returns {string} - The final public URL string. + */ +export function getPublicUrl(publicUrl, req) { if (publicUrl) { return publicUrl; } return getUrlObject(req).toString(); -}; +} -export const getTileUrls = ( +/** + * Generates an array of tile URLs based on given parameters. + * @param {object} req - Express request object. + * @param {string | string[]} domains - Domain(s) to use for tile URLs. + * @param {string} path - The base path for the tiles. + * @param {number} [tileSize] - The size of the tile (optional). + * @param {string} format - The format of the tiles (e.g., 'png', 'jpg'). + * @param {string} publicUrl - The public URL to use (if not using domains). + * @param {object} [aliases] - Aliases for format extensions. + * @returns {string[]} An array of tile URL strings. + */ +export function getTileUrls( req, domains, path, @@ -79,7 +97,7 @@ export const getTileUrls = ( format, publicUrl, aliases, -) => { +) { const urlObject = getUrlObject(req); if (domains) { if (domains.constructor === String && domains.length > 0) { @@ -144,9 +162,14 @@ export const getTileUrls = ( } return uris; -}; +} -export const fixTileJSONCenter = (tileJSON) => { +/** + * Fixes the center in the tileJSON if no center is available. + * @param {object} tileJSON - The tileJSON object to process. + * @returns {void} + */ +export function fixTileJSONCenter(tileJSON) { if (tileJSON.bounds && !tileJSON.center) { const fitWidth = 1024; const tiles = fitWidth / 256; @@ -159,10 +182,19 @@ export const fixTileJSONCenter = (tileJSON) => { ), ]; } -}; +} -const getFontPbf = (allowedFonts, fontPath, name, range, fallbacks) => - new Promise((resolve, reject) => { +/** + * Retrieves font data for a given font and range. + * @param {object} allowedFonts - An object of allowed fonts. + * @param {string} fontPath - The path to the font directory. + * @param {string} name - The name of the font. + * @param {string} range - The range (e.g., '0-255') of the font to load. + * @param {object} [fallbacks] - Optional fallback font list. + * @returns {Promise} A promise that resolves with the font data Buffer or rejects with an error. + */ +function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { + return new Promise((resolve, reject) => { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { const filename = path.join(fontPath, name, `${range}.pbf`); if (!fallbacks) { @@ -204,14 +236,24 @@ const getFontPbf = (allowedFonts, fontPath, name, range, fallbacks) => reject(`Font not allowed: ${name}`); } }); +} -export const getFontsPbf = async ( +/** + * Combines multiple font pbf buffers into one. + * @param {object} allowedFonts - An object of allowed fonts. + * @param {string} fontPath - The path to the font directory. + * @param {string} names - Comma-separated font names. + * @param {string} range - The range of the font (e.g., '0-255'). + * @param {object} [fallbacks] - Fallback font list. + * @returns {Promise} - A promise that resolves to the combined font data buffer. + */ +export async function getFontsPbf( allowedFonts, fontPath, names, range, fallbacks, -) => { +) { const fonts = names.split(','); const queue = []; for (const font of fonts) { @@ -228,9 +270,14 @@ export const getFontsPbf = async ( const combined = combine(await Promise.all(queue), names); return Buffer.from(combined.buffer, 0, combined.buffer.length); -}; +} -export const listFonts = async (fontPath) => { +/** + * Lists available fonts in a given font directory. + * @param {string} fontPath - The path to the font directory. + * @returns {Promise} - Promise that resolves with an object where keys are the font names. + */ +export async function listFonts(fontPath) { const existingFonts = {}; const files = await fsPromises.readdir(fontPath); @@ -245,9 +292,14 @@ export const listFonts = async (fontPath) => { } return existingFonts; -}; +} -export const isValidHttpUrl = (string) => { +/** + * Checks if a string is a valid HTTP or HTTPS URL. + * @param {string} string - The string to validate. + * @returns {boolean} True if the string is a valid HTTP/HTTPS URL, false otherwise. + */ +export function isValidHttpUrl(string) { let url; try { @@ -257,4 +309,4 @@ export const isValidHttpUrl = (string) => { } return url.protocol === 'http:' || url.protocol === 'https:'; -}; +} From 40ecf4cf762f78004761ce03c4bacb5c58044e72 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 22:15:33 -0500 Subject: [PATCH 030/104] Use common app.get for images and static images Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 508 +++++++++++++++++++++++------------------- 1 file changed, 273 insertions(+), 235 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index f3b5d94e5..39649477b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -624,266 +624,304 @@ const respondImage = async ( }); }; -const existingFonts = {}; -let maxScaleFactor = 2; - -export const serve_rendered = { - /** - * Initializes the serve_rendered module. - * @param {object} options Configuration options. - * @param {object} repo Repository object. - * @returns {Promise} A promise that resolves to the Express app. - */ - init: async function (options, repo) { - maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); - const app = express().disable('x-powered-by'); - - app.get( - `/:id{/:tileSize}/:z/:x/:y{@:scale}{.:format}`, - async (req, res, next) => { - try { - console.log(req.params); - if ( - req.params.z === 'static' || - (req.params.tileSize && - req.params.tileSize != 256 && - req.params.tileSize != 512) - ) { - //workaroud for /:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format} - next('route'); - } else { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); - } +/** + * Handles requests for tile images. + * @param {object} options - Configuration options for the server. + * @param {object} repo - The repository object holding style data. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {Function} next - Express next middleware function. + * @param {number} maxScaleFactor - The maximum scale factor allowed. + * @returns {Promise} + */ +async function handleTileRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, +) { + const { + id, + p2: zParam, + p3: xParam, + p4: yParam, + scale: scaleParam, + format, + p1: tileSize, + } = req.params; + const item = repo[id]; + if (!item) { + return res.sendStatus(404); + } - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(item.lastModified) <= new Date(modifiedSince)) { - return res.sendStatus(304); - } - } + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if (new Date(item.lastModified) <= new Date(modifiedSince)) { + return res.sendStatus(304); + } + } + const z = parseFloat(zParam) | 0; + const x = parseFloat(xParam) | 0; + const y = parseFloat(yParam) | 0; + const scale = parseScale(scaleParam, maxScaleFactor); + const parsedTileSize = parseInt(tileSize, 10) || 256; + if ( + scale == null || + z < 0 || + x < 0 || + y < 0 || + z > 22 || + x >= Math.pow(2, z) || + y >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - const scale = parseScale(req.params.scale, maxScaleFactor); - const format = req.params.format; - const tileSize = parseInt(req.params.tileSize, 10) || 256; - if ( - scale == null || - z < 0 || - x < 0 || - y < 0 || - z > 22 || - x >= Math.pow(2, z) || - y >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } + const tileCenter = mercator.ll( + [((x + 0.5) / (1 << z)) * (256 << z), ((y + 0.5) / (1 << z)) * (256 << z)], + z, + ); - const tileCenter = mercator.ll( - [ - ((x + 0.5) / (1 << z)) * (256 << z), - ((y + 0.5) / (1 << z)) * (256 << z), - ], - z, - ); - - // prettier-ignore - return await respondImage( - options, item, z, tileCenter[0], tileCenter[1], 0, 0, tileSize, tileSize, scale, format, res, - ); - } - } catch (e) { - console.log(e); - next('route'); - } - }, - ); + // prettier-ignore + return await respondImage( + options, item, z, tileCenter[0], tileCenter[1], 0, 0, parsedTileSize, parsedTileSize, scale, format, res, + ); +} - if (options.serveStaticMaps !== false) { - app.get( - `/:id/static{/:raw}{/:type}/:width{x:height}{@:scale}{.:format}`, - async (req, res, next) => { - try { - const item = repo[req.params.id]; - console.log(req.params); - const format = req.params.format; - const w = parseInt(req.params.width) || 512; - const h = parseInt(req.params.height) || 512; - const scale = parseScale(req.params.scale, maxScaleFactor); - let raw = req.params.raw !== undefined; - let type = req.params.type; - if (!type) { - //workaround for type when raw is not set - type = req.params.raw; - raw = false; - } +/** + * Handles requests for static map images. + * @param {object} options - Configuration options for the server. + * @param {object} repo - The repository object holding style data. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {Function} next - Express next middleware function. + * @param {number} maxScaleFactor - The maximum scale factor allowed. + * @returns {Promise} + */ +async function handleStaticRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, +) { + const { + id, + scale: scaleParam, + format, + p2: raw, + p3: type, + p4: width, + p5: height, + } = req.params; + const item = repo[id]; + const parsedWidth = parseInt(width) || 512; + const parsedHeight = parseInt(height) || 512; + const scale = parseScale(scaleParam, maxScaleFactor); + let isRaw = raw !== undefined; + let staticType = type; + + if (!staticType) { + //workaround for type when raw is not set + staticType = raw; + isRaw = false; + } - if (!item || !type || !format || !scale) { - return res.sendStatus(404); - } + if (!item || !staticType || !format || !scale) { + return res.sendStatus(404); + } - const staticTypeMatch = type.match(staticTypeRegex); - console.log(staticTypeMatch.groups); - if (staticTypeMatch.groups.lon) { - // Center Based Static Image - const z = staticTypeMatch.groups.zoom; - let x = staticTypeMatch.groups.lon; - let y = staticTypeMatch.groups.lat; - const bearing = staticTypeMatch.groups.bearing; - const pitch = staticTypeMatch.groups.pitch; - - if (z < 0) { - return res.status(404).send('Invalid zoom'); - } + const staticTypeMatch = staticType.match(staticTypeRegex); + console.log(staticTypeMatch); + if (staticTypeMatch.groups.lon) { + // Center Based Static Image + const z = parseFloat(staticTypeMatch.groups.zoom) || 0; + let x = parseFloat(staticTypeMatch.groups.lon) || 0; + let y = parseFloat(staticTypeMatch.groups.lat) || 0; + const bearing = parseFloat(staticTypeMatch.groups.bearing) || 0; + const pitch = parseInt(staticTypeMatch.groups.pitch) || 0; + if (z < 0) { + return res.status(404).send('Invalid zoom'); + } - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; + const transformer = isRaw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; - if (transformer) { - const ll = transformer([x, y]); - x = ll[0]; - y = ll[1]; - } + if (transformer) { + const ll = transformer([x, y]); + x = ll[0]; + y = ll[1]; + } - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery(req.query, options, transformer); - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); - // prettier-ignore - return await respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } else if (staticTypeMatch.groups.minx) { - // Area Based Static Image - const bbox = [ - +staticTypeMatch.groups.minx, - +staticTypeMatch.groups.miny, - +staticTypeMatch.groups.maxx, - +staticTypeMatch.groups.maxx, - ]; - let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; - - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - if (transformer) { - const minCorner = transformer(bbox.slice(0, 2)); - const maxCorner = transformer(bbox.slice(2)); - bbox[0] = minCorner[0]; - bbox[1] = minCorner[1]; - bbox[2] = maxCorner[0]; - bbox[3] = maxCorner[1]; - center = transformer(center); - } + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.minx) { + // Area Based Static Image + const bbox = [ + +staticTypeMatch.groups.minx, + +staticTypeMatch.groups.miny, + +staticTypeMatch.groups.maxx, + +staticTypeMatch.groups.maxx, + ]; + let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; + + const transformer = isRaw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + if (transformer) { + const minCorner = transformer(bbox.slice(0, 2)); + const maxCorner = transformer(bbox.slice(2)); + bbox[0] = minCorner[0]; + bbox[1] = minCorner[1]; + bbox[2] = maxCorner[0]; + bbox[3] = maxCorner[1]; + center = transformer(center); + } - const z = calcZForBBox(bbox, w, h, req.query); - const x = center[0]; - const y = center[1]; - const bearing = 0; - const pitch = 0; + const z = calcZForBBox(bbox, parsedWidth, parsedHeight, req.query); + const x = center[0]; + const y = center[1]; + const bearing = 0; + const pitch = 0; + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery(req.query, options, transformer); + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); + } else if (staticTypeMatch.groups.auto) { + // Area Static Image + const bearing = 0; + const pitch = 0; + + const transformer = isRaw + ? mercator.inverse.bind(mercator) + : item.dataProjWGStoInternalWGS; + + const paths = extractPathsFromQuery(req.query, transformer); + const markers = extractMarkersFromQuery(req.query, options, transformer); + + // Extract coordinates from markers + const markerCoordinates = []; + for (const marker of markers) { + markerCoordinates.push(marker.location); + } - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); + // Create array with coordinates from markers and path + const coords = [].concat(paths.flat()).concat(markerCoordinates); - // prettier-ignore - return await respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } else if (staticTypeMatch.groups.auto) { - // Area Static Image - const bearing = 0; - const pitch = 0; - - const transformer = raw - ? mercator.inverse.bind(mercator) - : item.dataProjWGStoInternalWGS; - - const paths = extractPathsFromQuery(req.query, transformer); - const markers = extractMarkersFromQuery( - req.query, - options, - transformer, - ); + // Check if we have at least one coordinate to calculate a bounding box + if (coords.length < 1) { + return res.status(400).send('No coordinates provided'); + } - // Extract coordinates from markers - const markerCoordinates = []; - for (const marker of markers) { - markerCoordinates.push(marker.location); - } + const bbox = [Infinity, Infinity, -Infinity, -Infinity]; + for (const pair of coords) { + bbox[0] = Math.min(bbox[0], pair[0]); + bbox[1] = Math.min(bbox[1], pair[1]); + bbox[2] = Math.max(bbox[2], pair[0]); + bbox[3] = Math.max(bbox[3], pair[1]); + } - // Create array with coordinates from markers and path - const coords = [].concat(paths.flat()).concat(markerCoordinates); + const bbox_ = mercator.convert(bbox, '900913'); + const center = mercator.inverse([ + (bbox_[0] + bbox_[2]) / 2, + (bbox_[1] + bbox_[3]) / 2, + ]); + + // Calculate zoom level + const maxZoom = parseFloat(req.query.maxzoom); + let z = calcZForBBox(bbox, parsedWidth, parsedHeight, req.query); + if (maxZoom > 0) { + z = Math.min(z, maxZoom); + } - // Check if we have at least one coordinate to calculate a bounding box - if (coords.length < 1) { - return res.status(400).send('No coordinates provided'); - } + const x = center[0]; + const y = center[1]; - const bbox = [Infinity, Infinity, -Infinity, -Infinity]; - for (const pair of coords) { - bbox[0] = Math.min(bbox[0], pair[0]); - bbox[1] = Math.min(bbox[1], pair[1]); - bbox[2] = Math.max(bbox[2], pair[0]); - bbox[3] = Math.max(bbox[3], pair[1]); - } + // prettier-ignore + const overlay = await renderOverlay( + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); - const bbox_ = mercator.convert(bbox, '900913'); - const center = mercator.inverse([ - (bbox_[0] + bbox_[2]) / 2, - (bbox_[1] + bbox_[3]) / 2, - ]); - - // Calculate zoom level - const maxZoom = parseFloat(req.query.maxzoom); - let z = calcZForBBox(bbox, w, h, req.query); - if (maxZoom > 0) { - z = Math.min(z, maxZoom); - } + // prettier-ignore + return await respondImage( + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); + } else { + return res.sendStatus(404); + } +} - const x = center[0]; - const y = center[1]; +const existingFonts = {}; +let maxScaleFactor = 2; - // prettier-ignore - const overlay = await renderOverlay( - z, x, y, bearing, pitch, w, h, scale, paths, markers, req.query, - ); +export const serve_rendered = { + /** + * Initializes the serve_rendered module. + * @param {object} options Configuration options. + * @param {object} repo Repository object. + * @returns {Promise} A promise that resolves to the Express app. + */ + init: async function (options, repo) { + maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); + const app = express().disable('x-powered-by'); - // prettier-ignore - return await respondImage( - options, item, z, x, y, bearing, pitch, w, h, scale, format, res, overlay, 'static', - ); - } else { - return res.sendStatus(404); + app.get( + `/:id{/:p1}/:p2/:p3/:p4{x:p5}{@:scale}{.:format}`, + async (req, res, next) => { + try { + const { p2 } = req.params; + if (p2 === 'static') { + // Route to static if p2 is static + if (options.serveStaticMaps !== false) { + return handleStaticRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, + ); } - } catch (e) { - next('route'); + return res.sendStatus(404); } - }, - ); - } + + return handleTileRequest( + options, + repo, + req, + res, + next, + maxScaleFactor, + ); + } catch (e) { + console.log(e); + return next(e); + } + }, + ); app.get('{/:tileSize}/:id.json', (req, res, next) => { const item = repo[req.params.id]; From d98cc3328f2b2684448c8b8eb95ae054bce3660d Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 22:17:46 -0500 Subject: [PATCH 031/104] add allowedTileSizes and option Co-Authored-By: Andrew Calcutt --- src/server.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/server.js b/src/server.js index 1dbad650d..afb9d2d35 100644 --- a/src/server.js +++ b/src/server.js @@ -16,7 +16,12 @@ import morgan from 'morgan'; import { serve_data } from './serve_data.js'; import { serve_style } from './serve_style.js'; import { serve_font } from './serve_font.js'; -import { getTileUrls, getPublicUrl, isValidHttpUrl } from './utils.js'; +import { + getTileUrls, + getPublicUrl, + isValidHttpUrl, + allowedOptions, +} from './utils.js'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); @@ -99,6 +104,10 @@ async function start(opts) { ? path.resolve(paths.root, paths.files) : path.resolve(__dirname, '../public/files'); + const allowedTileSizes = allowedOptions(['256', '512'], { + defaultValue: options.tileSize || 256, + }); + const startupPromises = []; for (const type of Object.keys(paths)) { @@ -389,17 +398,19 @@ async function start(opts) { } app.get('{/:tileSize}/rendered.json', (req, res, next) => { - const tileSize = parseInt(req.params.tileSize, 10) || undefined; - res.send(addTileJSONs([], req, 'rendered', tileSize)); + const tileSize = allowedTileSizes(req.params['tileSize']); + res.send(addTileJSONs([], req, 'rendered', parseInt(tileSize, 10))); }); + app.get('/data.json', (req, res) => { res.send(addTileJSONs([], req, 'data', undefined)); }); + app.get('{/:tileSize}/index.json', (req, res, next) => { - const tileSize = parseInt(req.params.tileSize, 10) || undefined; + const tileSize = allowedTileSizes(req.params['tileSize']); res.send( addTileJSONs( - addTileJSONs([], req, 'rendered', tileSize), + addTileJSONs([], req, 'rendered', parseInt(tileSize, 10)), req, 'data', undefined, From 4c6f379f4ec59405468cd9cbb48ef64941b23e28 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 29 Dec 2024 22:18:02 -0500 Subject: [PATCH 032/104] cleanup error responses Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 69 +++++++++++++++++++++++----------------------- src/utils.js | 16 +++++++++-- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index d0c2e3b82..fbdda7c66 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -19,7 +19,7 @@ const allowedSpriteFormats = allowedOptions(['png', 'json']); */ function allowedSpriteScales(scale) { if (!scale) return ''; - const match = scale.match(/(\d+)x/); + const match = scale.match(/(\d+)x/); const parsedScale = match ? parseInt(match[1], 10) : 1; return '@' + Math.min(parsedScale, 3) + 'x'; } @@ -64,42 +64,43 @@ export const serve_style = { }); app.get(`/:id/:sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { - console.log(req.params); const { spriteID = 'default', id, format } = req.params; const scale = allowedSpriteScales(req.params.scale); - try { - if ( - !allowedSpriteFormats(format) || - ((id == 256 || id == 512) && format === 'json') - ) { - //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml - next('route'); - } else { - const item = repo[id]; - const sprite = item.spritePaths.find( - (sprite) => sprite.id === spriteID, - ); - if (sprite) { - const filename = `${sprite.path + scale}.${format}`; - return fs.readFile(filename, (err, data) => { - if (err) { - console.log('Sprite load error:', filename); - return res.sendStatus(404); - } else { - if (format === 'json') - res.header('Content-type', 'application/json'); - if (format === 'png') res.header('Content-type', 'image/png'); - return res.send(data); - } - }); - } else { - return res.status(400).send('Bad Sprite ID or Scale'); - } - } - } catch (e) { - console.log(e); - next('route'); + + if ( + !allowedSpriteFormats(format) || + ((id == 256 || id == 512) && format === 'json') + ) { + //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml + return next('route'); } + + const item = repo[id]; + if (!item) { + return res.sendStatus(404); // Ensure item exists first to prevent errors + } + + const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); + if (!sprite) { + return res.status(400).send('Bad Sprite ID or Scale'); + } + + const spriteScale = allowedSpriteScales(scale); + const filename = `${sprite.path}${spriteScale}.${format}`; + + fs.readFile(filename, (err, data) => { + if (err) { + console.error('Sprite load error: %s, Error: %s', filename, err); + return res.sendStatus(404); + } + + if (format === 'json') { + res.header('Content-type', 'application/json'); + } else if (format === 'png') { + res.header('Content-type', 'image/png'); + } + return res.send(data); + }); }); return app; diff --git a/src/utils.js b/src/utils.js index 158f052f4..311f8d00c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -196,14 +196,23 @@ export function fixTileJSONCenter(tileJSON) { function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return new Promise((resolve, reject) => { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { + if (!name || typeof name !== 'string' || name.trim() === '') { + console.error('ERROR: Invalid font name: %s', name); + return reject('Invalid font name'); + } + if (!/^\d+-\d+$/.test(range)) { + console.error('ERROR: Invalid range: %s', range); + return reject('Invalid range'); + } const filename = path.join(fontPath, name, `${range}.pbf`); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } delete fallbacks[name]; + // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { - console.error(`ERROR: Font not found: ${name}`); + console.error('ERROR: Font not found: %s, Error: %s', filename, err); if (fallbacks && Object.keys(fallbacks).length) { let fallbackName; @@ -219,7 +228,10 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } } - console.error(`ERROR: Trying to use ${fallbackName} as a fallback`); + console.error( + `ERROR: Trying to use %s as a fallback`, + fallbackName, + ); delete fallbacks[fallbackName]; getFontPbf(null, fontPath, fallbackName, range, fallbacks).then( resolve, From 592e74c303708f90f85a3a3e875316523e707503 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 00:22:14 -0500 Subject: [PATCH 033/104] fix /style/id.json with next('route') Co-Authored-By: Andrew Calcutt --- src/server.js | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/server.js b/src/server.js index afb9d2d35..b47ebeb60 100644 --- a/src/server.js +++ b/src/server.js @@ -445,33 +445,40 @@ async function start(opts) { try { const content = fs.readFileSync(templateFile, 'utf-8'); const compiled = handlebars.compile(content.toString()); - app.get(urlPath, (req, res) => { - console.log(`Serving template at path: ${urlPath}`); + app.get(urlPath, (req, res, next) => { + if (opts.verbose) { + console.log(`Serving template at path: ${urlPath}`); + } let data = {}; if (dataGetter) { data = dataGetter(req); - if (!data) { - console.error(`Data getter for ${template} returned null`); - return res.status(404).send('Not found'); + if (data) { + data['server_version'] = + `${packageJson.name} v${packageJson.version}`; + data['public_url'] = opts.publicUrl || '/'; + data['is_light'] = isLight; + data['key_query_part'] = req.query.key + ? `key=${encodeURIComponent(req.query.key)}&` + : ''; + data['key_query'] = req.query.key + ? `?key=${encodeURIComponent(req.query.key)}` + : ''; + if (template === 'wmts') res.set('Content-Type', 'text/xml'); + return res.status(200).send(compiled(data)); + } else { + if (opts.verbose) { + console.log(`Forwarding request for: ${urlPath} to next route`); + } + next('route'); } } - data['server_version'] = `${packageJson.name} v${packageJson.version}`; - data['public_url'] = opts.publicUrl || '/'; - data['is_light'] = isLight; - data['key_query_part'] = req.query.key - ? `key=${encodeURIComponent(req.query.key)}&` - : ''; - data['key_query'] = req.query.key - ? `?key=${encodeURIComponent(req.query.key)}` - : ''; - if (template === 'wmts') res.set('Content-Type', 'text/xml'); - return res.status(200).send(compiled(data)); }); } catch (err) { console.error(`Error reading template file: ${templateFile}`, err); throw new Error(`Template not found: ${err.message}`); //throw an error so that the server doesnt start } } + serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { From 32fd48814cc39a7796626792cea55f965355368a Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 00:23:33 -0500 Subject: [PATCH 034/104] improve sprite path Co-Authored-By: Andrew Calcutt --- src/serve_style.js | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index fbdda7c66..c26eb94a7 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -63,21 +63,13 @@ export const serve_style = { return res.send(styleJSON_); }); - app.get(`/:id/:sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { const { spriteID = 'default', id, format } = req.params; - const scale = allowedSpriteScales(req.params.scale); - - if ( - !allowedSpriteFormats(format) || - ((id == 256 || id == 512) && format === 'json') - ) { - //Workaround for {/:tileSize}/:id.json' and /styles/:id/wmts.xml - return next('route'); - } + const spriteScale = allowedSpriteScales(req.params.scale); const item = repo[id]; - if (!item) { - return res.sendStatus(404); // Ensure item exists first to prevent errors + if (!item || !allowedSpriteFormats(format)) { + return res.sendStatus(404); } const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); @@ -85,9 +77,9 @@ export const serve_style = { return res.status(400).send('Bad Sprite ID or Scale'); } - const spriteScale = allowedSpriteScales(scale); const filename = `${sprite.path}${spriteScale}.${format}`; + // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { console.error('Sprite load error: %s, Error: %s', filename, err); From 576f02eb0b1d4849c356b9bc6daf525dc328026f Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 00:24:03 -0500 Subject: [PATCH 035/104] add parseFloadts around zxy Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 245ef004e..c3e03e215 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -36,9 +36,9 @@ export const serve_data = { return res.sendStatus(404); } const tileJSONFormat = item.tileJSON.format; - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; + const z = parseFloat(req.params.z) | 0; + const x = parseFloat(req.params.x) | 0; + const y = parseFloat(req.params.y) | 0; let format = req.params.format; if (format === options.pbfAlias) { format = 'pbf'; From d6f7f5e926fc83c41f925aafa8cecb177da3645e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 02:39:47 -0500 Subject: [PATCH 036/104] simplify server_data Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 309 ++++++++++------------------------------------ 1 file changed, 67 insertions(+), 242 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index c3e03e215..c951bcd6e 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -36,9 +36,13 @@ export const serve_data = { return res.sendStatus(404); } const tileJSONFormat = item.tileJSON.format; - const z = parseFloat(req.params.z) | 0; - const x = parseFloat(req.params.x) | 0; - const y = parseFloat(req.params.y) | 0; + const z = parseInt(req.params.z, 10); + const x = parseInt(req.params.x, 10); + const y = parseInt(req.params.y, 10); + if (isNaN(z) || isNaN(x) || isNaN(y)) { + return res.status(404).send('Invalid Tile'); + } + let format = req.params.format; if (format === options.pbfAlias) { format = 'pbf'; @@ -51,7 +55,6 @@ export const serve_data = { } if ( z < item.tileJSON.minzoom || - 0 || x < 0 || y < 0 || z > item.tileJSON.maxzoom || @@ -60,255 +63,77 @@ export const serve_data = { ) { return res.status(404).send('Out of bounds'); } - if (item.sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(item.source, z, x, y); - if (tileinfo == undefined || tileinfo.data == undefined) { - return res.status(404).send('Not found'); - } else { - let data = tileinfo.data; - let headers = tileinfo.header; - if (tileJSONFormat === 'pbf') { - if (options.dataDecoratorFunc) { - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); - } - } - data = JSON.stringify(geojson); - } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); - data = await gzipP(data); - - return res.status(200).send(data); - } + let getTile; + if (item.sourceType === 'pmtiles') { + const tileinfo = await getPMtilesTile(item.source, z, x, y); + if (!tileinfo?.data) return res.status(204).send(); + getTile = { data: tileinfo.data, header: tileinfo.header }; } else if (item.sourceType === 'mbtiles') { - item.source.getTile(z, x, y, async (err, data, headers) => { - let isGzipped; - if (err) { - if (/does not exist/.test(err.message)) { - return res.status(204).send(); - } else { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } - } else { - if (data == null) { - return res.status(404).send('Not found'); - } else { - if (tileJSONFormat === 'pbf') { - isGzipped = - data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - if (options.dataDecoratorFunc) { - if (isGzipped) { - data = await gunzipP(data); - isGzipped = false; - } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - - app.get('/:id/elevation/:z/:x/:y', - async (req, res, next) => { try { - const item = repo?.[req.params.id]; - if (!item) return res.sendStatus(404); - if (!item.source) return res.status(404).send('Missing source'); - if (!item.tileJSON) return res.status(404).send('Missing tileJSON'); - if (!item.sourceType) - return res.status(404).send('Missing sourceType'); - - const { source, tileJSON, sourceType } = item; - - if (sourceType !== 'pmtiles' && sourceType !== 'mbtiles') { - return res - .status(400) - .send('Invalid sourceType. Must be pmtiles or mbtiles.'); - } - - const encoding = tileJSON?.encoding; - if (encoding == null) { - return res.status(400).send('Missing tileJSON.encoding'); - } else if (encoding !== 'terrarium' && encoding !== 'mapbox') { - return res - .status(400) - .send('Invalid encoding. Must be terrarium or mapbox.'); - } - - const format = tileJSON?.format; - if (format == null) { - return res.status(400).send('Missing tileJSON.format'); - } else if (format !== 'webp' && format !== 'png') { - return res.status(400).send('Invalid format. Must be webp or png.'); - } - - const z = parseInt(req.params.z, 10); - const x = parseFloat(req.params.x); - const y = parseFloat(req.params.y); - - if (tileJSON.minzoom == null || tileJSON.maxzoom == null) { - return res.status(404).send(JSON.stringify(tileJSON)); - } - - const TILE_SIZE = 256; - let tileCenter; - let xy; - - if (Number.isInteger(x) && Number.isInteger(y)) { - const intX = parseInt(req.params.x, 10); - const intY = parseInt(req.params.y, 10); + getTile = await new Promise((resolve, reject) => { + item.source.getTile(z, x, y, (err, tileData, tileHeader) => { + if (err) { + return /does not exist/.test(err.message) + ? resolve(null) + : reject(err); + } + resolve({ data: tileData, header: tileHeader }); + }); + }); + } catch (e) { + return res.status(500).send(e.message); + } + } + if (getTile == null) return res.status(204).send(); - if ( - z < tileJSON.minzoom || - z > tileJSON.maxzoom || - intX < 0 || - intY < 0 || - intX >= Math.pow(2, z) || - intY >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } - xy = [intX, intY]; - tileCenter = new SphericalMercator().bbox(intX, intY, z); - } else { - if ( - z < tileJSON.minzoom || - z > tileJSON.maxzoom || - x < -180 || - y < -90 || - x > 180 || - y > 90 - ) { - return res.status(404).send('Out of bounds'); - } + let data = getTile.data; + let headers = getTile.header; + let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - tileCenter = [y, x, y + 0.1, x + 0.1]; - const { minX, minY } = new SphericalMercator().xyz(tileCenter, z); - xy = [minX, minY]; + if (tileJSONFormat === 'pbf') { + if (options.dataDecoratorFunc) { + if (isGzipped) { + data = await gunzipP(data); + isGzipped = false; } + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + } + } - let data; - if (sourceType === 'pmtiles') { - const tileinfo = await getPMtilesTile(source, z, x, y); - if (!tileinfo?.data) return res.status(204).send(); - data = tileinfo.data; - } else { - data = await new Promise((resolve, reject) => { - source.getTile(z, xy[0], xy[1], (err, tileData) => { - if (err) { - return /does not exist/.test(err.message) - ? resolve(null) - : reject(err); - } - resolve(tileData); - }); - }); + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); } - if (data == null) return res.status(204).send(); - if (!data) return res.status(404).send('Not found'); - - const image = new Image(); - await new Promise(async (resolve, reject) => { - image.onload = async () => { - const canvas = createCanvas(TILE_SIZE, TILE_SIZE); - const context = canvas.getContext('2d'); - context.drawImage(image, 0, 0); - const imgdata = context.getImageData(0, 0, TILE_SIZE, TILE_SIZE); - - const arrayWidth = imgdata.width; - const arrayHeight = imgdata.height; - const bytesPerPixel = 4; - - const xPixel = Math.floor(xy[0]); - const yPixel = Math.floor(xy[1]); - - if ( - xPixel < 0 || - yPixel < 0 || - xPixel >= arrayWidth || - yPixel >= arrayHeight - ) { - return reject('Out of bounds Pixel'); - } - - const index = (yPixel * arrayWidth + xPixel) * bytesPerPixel; - - const red = imgdata.data[index]; - const green = imgdata.data[index + 1]; - const blue = imgdata.data[index + 2]; - - let elevation; - if (encoding === 'mapbox') { - elevation = - -10000 + (red * 256 * 256 + green * 256 + blue) * 0.1; - } else if (encoding === 'terrarium') { - elevation = red * 256 + green + blue / 256 - 32768; - } else { - elevation = 'invalid encoding'; - } - - resolve( - res.status(200).send({ - z, - x: xy[0], - y: xy[1], - red, - green, - blue, - latitude: tileCenter[0], - longitude: tileCenter[1], - elevation, - }), - ); - }; + } + data = JSON.stringify(geojson); + } + console.log(headers); + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); - image.onerror = (err) => reject(err); + if (!isGzipped) { + data = await gzipP(data); + } - if (format === 'webp') { - try { - const img = await sharp(data).toFormat('png').toBuffer(); - image.src = img; - } catch (err) { - reject(err); - } - } else { - image.src = data; - } - }); - } catch (err) { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } - }, - ); + return res.status(200).send(data); + }); - app.get('/:id.json', (req, res, next) => { + app.get('/:id.json', (req, res) => { const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); From 99afa33e9df5f6f06e12fd2e0ecd4e259cb8b3ce Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 11:12:20 -0500 Subject: [PATCH 037/104] move tile fetch and add fix verbose logging Co-Authored-By: Andrew Calcutt --- src/serve_data.js | 47 ++++++-------- src/serve_rendered.js | 140 ++++++++++++++++-------------------------- src/serve_style.js | 5 +- src/server.js | 8 +-- src/utils.js | 29 +++++++++ 5 files changed, 108 insertions(+), 121 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index c951bcd6e..84b9234d8 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -11,7 +11,12 @@ import SphericalMercator from '@mapbox/sphericalmercator'; import { Image, createCanvas } from 'canvas'; import sharp from 'sharp'; -import { fixTileJSONCenter, getTileUrls, isValidHttpUrl } from './utils.js'; +import { + fixTileJSONCenter, + getTileUrls, + isValidHttpUrl, + fetchTileData, +} from './utils.js'; import { getPMtilesInfo, getPMtilesTile, @@ -64,31 +69,17 @@ export const serve_data = { return res.status(404).send('Out of bounds'); } - let getTile; - if (item.sourceType === 'pmtiles') { - const tileinfo = await getPMtilesTile(item.source, z, x, y); - if (!tileinfo?.data) return res.status(204).send(); - getTile = { data: tileinfo.data, header: tileinfo.header }; - } else if (item.sourceType === 'mbtiles') { - try { - getTile = await new Promise((resolve, reject) => { - item.source.getTile(z, x, y, (err, tileData, tileHeader) => { - if (err) { - return /does not exist/.test(err.message) - ? resolve(null) - : reject(err); - } - resolve({ data: tileData, header: tileHeader }); - }); - }); - } catch (e) { - return res.status(500).send(e.message); - } - } - if (getTile == null) return res.status(204).send(); + const fetchTile = await fetchTileData( + item.source, + item.sourceType, + z, + x, + y, + ); + if (fetchTile == null) return res.status(204).send(); - let data = getTile.data; - let headers = getTile.header; + let data = fetchTile.data; + let headers = fetchTile.headers; let isGzipped = data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; if (tileJSONFormat === 'pbf') { @@ -121,7 +112,6 @@ export const serve_data = { } data = JSON.stringify(geojson); } - console.log(headers); delete headers['ETag']; // do not trust the tile ETag -- regenerate headers['Content-Encoding'] = 'gzip'; res.set(headers); @@ -162,10 +152,11 @@ export const serve_data = { * @param {object} repo Repository object. * @param {object} params Parameters object. * @param {string} id ID of the data source. - * @param {string} publicUrl Public URL of the data. + * @param {object} programOpts - An object containing the program options * @returns {Promise} */ - add: async function (options, repo, params, id, publicUrl) { + add: async function (options, repo, params, id, programOpts) { + const { publicUrl } = programOpts; let inputFile; let inputType; if (params.pmtiles) { diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 39649477b..944b8410b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -33,12 +33,9 @@ import { getTileUrls, isValidHttpUrl, fixTileJSONCenter, + fetchTileData, } from './utils.js'; -import { - openPMtiles, - getPMtilesInfo, - getPMtilesTile, -} from './pmtiles_adapter.js'; +import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; import { renderOverlay, renderWatermark, renderAttribution } from './render.js'; import fsp from 'node:fs/promises'; import { existsP, gunzipP } from './promises.js'; @@ -951,11 +948,11 @@ export const serve_rendered = { * @param {object} repo Repository object. * @param {object} params Parameters object. * @param {string} id ID of the item. - * @param {string} publicUrl Public URL. + * @param {object} programOpts - An object containing the program options * @param {Function} dataResolver Function to resolve data. * @returns {Promise} */ - add: async function (options, repo, params, id, publicUrl, dataResolver) { + add: async function (options, repo, params, id, programOpts, dataResolver) { const map = { renderers: [], renderersStatic: [], @@ -963,6 +960,8 @@ export const serve_rendered = { sourceTypes: {}, }; + const { publicUrl, verbose } = programOpts; + let styleJSON; /** * Creates a pool of renderers. @@ -1023,88 +1022,57 @@ export const serve_rendered = { const y = parts[5].split('.')[0] | 0; const format = parts[5].split('.')[1]; - if (sourceType === 'pmtiles') { - let tileinfo = await getPMtilesTile(source, z, x, y); - let data = tileinfo.data; - let headers = tileinfo.header; - if (data == undefined) { - if (options.verbose) - console.log('MBTiles error, serving empty', err); - createEmptyResponse( - sourceInfo.format, - sourceInfo.color, - callback, + const fetchTile = await fetchTileData( + source, + sourceType, + z, + x, + y, + ); + if (fetchTile == null) { + if (verbose) { + console.log( + 'fetchTile error on %s, serving empty response', + req.url, ); - return; - } else { - const response = {}; - response.data = data; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } - } - - callback(null, response); } - } else if (sourceType === 'mbtiles') { - source.getTile(z, x, y, async (err, data, headers) => { - if (err) { - if (options.verbose) - console.log('MBTiles error, serving empty', err); - createEmptyResponse( - sourceInfo.format, - sourceInfo.color, - callback, - ); - return; - } - - const response = {}; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - try { - response.data = await gunzipP(data); - } catch (err) { - console.log( - 'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf', - id, - z, - x, - y, - ); - } - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } - } else { - response.data = data; - } - - callback(null, response); - }); + createEmptyResponse( + sourceInfo.format, + sourceInfo.color, + callback, + ); + return; + } + + const response = {}; + response.data = fetchTile.data; + let headers = fetchTile.headers; + let isGzipped = + response.data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === + 0; + + if (headers['Last-Modified']) { + response.modified = new Date(headers['Last-Modified']); } + + if (format === 'pbf') { + if (isGzipped) { + response.data = await gunzipP(response.data); + isGzipped = false; + } + if (options.dataDecoratorFunc) { + response.data = options.dataDecoratorFunc( + sourceId, + 'data', + response.data, + z, + x, + y, + ); + } + } + + callback(null, response); } else if (protocol === 'http' || protocol === 'https') { try { const response = await axios.get(req.url, { diff --git a/src/serve_style.js b/src/serve_style.js index c26eb94a7..5a7478288 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -112,7 +112,7 @@ export const serve_style = { * @param {object} repo Repository object. * @param {object} params Parameters object containing style path * @param {string} id ID of the style. - * @param {string} publicUrl Public URL of the data. + * @param {object} programOpts - An object containing the program options * @param {Function} reportTiles Function for reporting tile sources. * @param {Function} reportFont Function for reporting font usage * @returns {boolean} true if add is succesful @@ -122,10 +122,11 @@ export const serve_style = { repo, params, id, - publicUrl, + programOpts, reportTiles, reportFont, ) { + const { publicUrl } = programOpts; const styleFile = path.resolve(options.paths.styles, params.style); let styleFileData; diff --git a/src/server.js b/src/server.js index b47ebeb60..df1134f89 100644 --- a/src/server.js +++ b/src/server.js @@ -193,7 +193,7 @@ async function start(opts) { serving.styles, item, id, - opts.publicUrl, + opts, (styleSourceId, protocol) => { let dataItemId; for (const id of Object.keys(data)) { @@ -250,7 +250,7 @@ async function start(opts) { serving.rendered, item, id, - opts.publicUrl, + opts, function dataResolver(styleSourceId) { let fileType; let inputFile; @@ -301,9 +301,7 @@ async function start(opts) { ); continue; } - startupPromises.push( - serve_data.add(options, serving.data, item, id, opts.publicUrl), - ); + startupPromises.push(serve_data.add(options, serving.data, item, id, opts)); } if (options.serveAllStyles) { fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => { diff --git a/src/utils.js b/src/utils.js index 311f8d00c..9a506c9b9 100644 --- a/src/utils.js +++ b/src/utils.js @@ -6,6 +6,7 @@ import fs from 'node:fs'; import clone from 'clone'; import { combine } from '@jsse/pbfont'; import { existsP } from './promises.js'; +import { getPMtilesTile } from './pmtiles_adapter.js'; /** * Restrict user input to an allowed set of options. @@ -322,3 +323,31 @@ export function isValidHttpUrl(string) { return url.protocol === 'http:' || url.protocol === 'https:'; } + +/** + * Fetches tile data from either PMTiles or MBTiles source. + * @param {object} source - The source object, which may contain a mbtiles object, or pmtiles object. + * @param {string} sourceType - The source type, which should be `pmtiles` or `mbtiles` + * @param {number} z - The zoom level. + * @param {number} x - The x coordinate of the tile. + * @param {number} y - The y coordinate of the tile. + * @returns {Promise} - A promise that resolves to an object with data and headers or null if no data is found. + */ +export async function fetchTileData(source, sourceType, z, x, y) { + if (sourceType === 'pmtiles') { + return await new Promise(async (resolve) => { + const tileinfo = await getPMtilesTile(source, z, x, y); + if (!tileinfo?.data) return resolve(null); + resolve({ data: tileinfo.data, headers: tileinfo.header }); + }); + } else if (sourceType === 'mbtiles') { + return await new Promise((resolve) => { + source.getTile(z, x, y, (err, tileData, tileHeader) => { + if (err) { + return resolve(null); + } + resolve({ data: tileData, headers: tileHeader }); + }); + }); + } +} From d5d938b732a06d4ff5a7537aa33136cf12205447 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Mon, 30 Dec 2024 11:22:30 -0500 Subject: [PATCH 038/104] add Handling request to verbose logging Co-Authored-By: Andrew Calcutt --- src/serve_rendered.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 944b8410b..1f298b1a3 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -984,7 +984,9 @@ export const serve_rendered = { ratio, request: async (req, callback) => { const protocol = req.url.split(':')[0]; - // console.log('Handling request:', req); + if (verbose) { + console.log('Handling request:', req); + } if (protocol === 'sprites') { const dir = options.paths[protocol]; const file = decodeURIComponent(req.url).substring( From 6fddbae15736551e4f11d329dc7e7e1b901f5da2 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Thu, 2 Jan 2025 18:17:19 -0500 Subject: [PATCH 039/104] merge elevation changes --- src/serve_data.js | 146 ++++++++++++++++++++++++++++++++++++++++++++++ src/server.js | 2 +- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/serve_data.js b/src/serve_data.js index 84b9234d8..ff896d057 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -123,6 +123,152 @@ export const serve_data = { return res.status(200).send(data); }); + app.get('/:id/elevation/:z/:x/:y', + async (req, res, next) => { + try { + const item = repo?.[req.params.id]; + if (!item) return res.sendStatus(404); + if (!item.source) return res.status(404).send('Missing source'); + if (!item.tileJSON) return res.status(404).send('Missing tileJSON'); + if (!item.sourceType) + return res.status(404).send('Missing sourceType'); + const { source, tileJSON, sourceType } = item; + if (sourceType !== 'pmtiles' && sourceType !== 'mbtiles') { + return res + .status(400) + .send('Invalid sourceType. Must be pmtiles or mbtiles.'); + } + const encoding = tileJSON?.encoding; + if (encoding == null) { + return res.status(400).send('Missing tileJSON.encoding'); + } else if (encoding !== 'terrarium' && encoding !== 'mapbox') { + return res + .status(400) + .send('Invalid encoding. Must be terrarium or mapbox.'); + } + const format = tileJSON?.format; + if (format == null) { + return res.status(400).send('Missing tileJSON.format'); + } else if (format !== 'webp' && format !== 'png') { + return res.status(400).send('Invalid format. Must be webp or png.'); + } + const z = parseInt(req.params.z, 10); + const x = parseFloat(req.params.x); + const y = parseFloat(req.params.y); + if (tileJSON.minzoom == null || tileJSON.maxzoom == null) { + return res.status(404).send(JSON.stringify(tileJSON)); + } + const TILE_SIZE = 256; + let tileCenter; + let xy; + if (Number.isInteger(x) && Number.isInteger(y)) { + const intX = parseInt(req.params.x, 10); + const intY = parseInt(req.params.y, 10); + if ( + z < tileJSON.minzoom || + z > tileJSON.maxzoom || + intX < 0 || + intY < 0 || + intX >= Math.pow(2, z) || + intY >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); + } + xy = [intX, intY]; + tileCenter = new SphericalMercator().bbox(intX, intY, z); + } else { + if ( + z < tileJSON.minzoom || + z > tileJSON.maxzoom || + x < -180 || + y < -90 || + x > 180 || + y > 90 + ) { + return res.status(404).send('Out of bounds'); + } + tileCenter = [y, x, y + 0.1, x + 0.1]; + const { minX, minY } = new SphericalMercator().xyz(tileCenter, z); + xy = [minX, minY]; + } + const fetchTile = await fetchTileData( + source, + sourceType, + z, + x, + y, + ); + if (fetchTile == null) return res.status(204).send(); + + let data = fetchTile.data; + const image = new Image(); + await new Promise(async (resolve, reject) => { + image.onload = async () => { + const canvas = createCanvas(TILE_SIZE, TILE_SIZE); + const context = canvas.getContext('2d'); + context.drawImage(image, 0, 0); + const imgdata = context.getImageData(0, 0, TILE_SIZE, TILE_SIZE); + const arrayWidth = imgdata.width; + const arrayHeight = imgdata.height; + const bytesPerPixel = 4; + const xPixel = Math.floor(xy[0]); + const yPixel = Math.floor(xy[1]); + if ( + xPixel < 0 || + yPixel < 0 || + xPixel >= arrayWidth || + yPixel >= arrayHeight + ) { + return reject('Out of bounds Pixel'); + } + const index = (yPixel * arrayWidth + xPixel) * bytesPerPixel; + const red = imgdata.data[index]; + const green = imgdata.data[index + 1]; + const blue = imgdata.data[index + 2]; + let elevation; + if (encoding === 'mapbox') { + elevation = + -10000 + (red * 256 * 256 + green * 256 + blue) * 0.1; + } else if (encoding === 'terrarium') { + elevation = red * 256 + green + blue / 256 - 32768; + } else { + elevation = 'invalid encoding'; + } + resolve( + res.status(200).send({ + z, + x: xy[0], + y: xy[1], + red, + green, + blue, + latitude: tileCenter[0], + longitude: tileCenter[1], + elevation, + }), + ); + }; + image.onerror = (err) => reject(err); + if (format === 'webp') { + try { + const img = await sharp(data).toFormat('png').toBuffer(); + image.src = img; + } catch (err) { + reject(err); + } + } else { + image.src = data; + } + }); + } catch (err) { + return res + .status(500) + .header('Content-Type', 'text/plain') + .send(err.message); + } + }, + ); + app.get('/:id.json', (req, res) => { const item = repo[req.params.id]; if (!item) { diff --git a/src/server.js b/src/server.js index df1134f89..beee4b36f 100644 --- a/src/server.js +++ b/src/server.js @@ -621,7 +621,7 @@ async function start(opts) { }; }); - serveTemplate('^/data{/:view}/:id/', 'data', (req) => { + serveTemplate('/data{/:view}/:id/', 'data', (req) => { const { id, view } = req.params; const data = serving.data[id]; From 8186aa1df0da1a54b36a4485d12c345b31ad55da Mon Sep 17 00:00:00 2001 From: acalcutt Date: Thu, 2 Jan 2025 18:23:52 -0500 Subject: [PATCH 040/104] lint format --- src/serve_data.js | 266 ++++++++++++++++++++++------------------------ 1 file changed, 128 insertions(+), 138 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index ff896d057..7d37f9018 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -123,151 +123,141 @@ export const serve_data = { return res.status(200).send(data); }); - app.get('/:id/elevation/:z/:x/:y', - async (req, res, next) => { - try { - const item = repo?.[req.params.id]; - if (!item) return res.sendStatus(404); - if (!item.source) return res.status(404).send('Missing source'); - if (!item.tileJSON) return res.status(404).send('Missing tileJSON'); - if (!item.sourceType) - return res.status(404).send('Missing sourceType'); - const { source, tileJSON, sourceType } = item; - if (sourceType !== 'pmtiles' && sourceType !== 'mbtiles') { - return res - .status(400) - .send('Invalid sourceType. Must be pmtiles or mbtiles.'); - } - const encoding = tileJSON?.encoding; - if (encoding == null) { - return res.status(400).send('Missing tileJSON.encoding'); - } else if (encoding !== 'terrarium' && encoding !== 'mapbox') { - return res - .status(400) - .send('Invalid encoding. Must be terrarium or mapbox.'); - } - const format = tileJSON?.format; - if (format == null) { - return res.status(400).send('Missing tileJSON.format'); - } else if (format !== 'webp' && format !== 'png') { - return res.status(400).send('Invalid format. Must be webp or png.'); + app.get('/:id/elevation/:z/:x/:y', async (req, res, next) => { + try { + const item = repo?.[req.params.id]; + if (!item) return res.sendStatus(404); + if (!item.source) return res.status(404).send('Missing source'); + if (!item.tileJSON) return res.status(404).send('Missing tileJSON'); + if (!item.sourceType) return res.status(404).send('Missing sourceType'); + const { source, tileJSON, sourceType } = item; + if (sourceType !== 'pmtiles' && sourceType !== 'mbtiles') { + return res + .status(400) + .send('Invalid sourceType. Must be pmtiles or mbtiles.'); + } + const encoding = tileJSON?.encoding; + if (encoding == null) { + return res.status(400).send('Missing tileJSON.encoding'); + } else if (encoding !== 'terrarium' && encoding !== 'mapbox') { + return res + .status(400) + .send('Invalid encoding. Must be terrarium or mapbox.'); + } + const format = tileJSON?.format; + if (format == null) { + return res.status(400).send('Missing tileJSON.format'); + } else if (format !== 'webp' && format !== 'png') { + return res.status(400).send('Invalid format. Must be webp or png.'); + } + const z = parseInt(req.params.z, 10); + const x = parseFloat(req.params.x); + const y = parseFloat(req.params.y); + if (tileJSON.minzoom == null || tileJSON.maxzoom == null) { + return res.status(404).send(JSON.stringify(tileJSON)); + } + const TILE_SIZE = 256; + let tileCenter; + let xy; + if (Number.isInteger(x) && Number.isInteger(y)) { + const intX = parseInt(req.params.x, 10); + const intY = parseInt(req.params.y, 10); + if ( + z < tileJSON.minzoom || + z > tileJSON.maxzoom || + intX < 0 || + intY < 0 || + intX >= Math.pow(2, z) || + intY >= Math.pow(2, z) + ) { + return res.status(404).send('Out of bounds'); } - const z = parseInt(req.params.z, 10); - const x = parseFloat(req.params.x); - const y = parseFloat(req.params.y); - if (tileJSON.minzoom == null || tileJSON.maxzoom == null) { - return res.status(404).send(JSON.stringify(tileJSON)); + xy = [intX, intY]; + tileCenter = new SphericalMercator().bbox(intX, intY, z); + } else { + if ( + z < tileJSON.minzoom || + z > tileJSON.maxzoom || + x < -180 || + y < -90 || + x > 180 || + y > 90 + ) { + return res.status(404).send('Out of bounds'); } - const TILE_SIZE = 256; - let tileCenter; - let xy; - if (Number.isInteger(x) && Number.isInteger(y)) { - const intX = parseInt(req.params.x, 10); - const intY = parseInt(req.params.y, 10); - if ( - z < tileJSON.minzoom || - z > tileJSON.maxzoom || - intX < 0 || - intY < 0 || - intX >= Math.pow(2, z) || - intY >= Math.pow(2, z) - ) { - return res.status(404).send('Out of bounds'); - } - xy = [intX, intY]; - tileCenter = new SphericalMercator().bbox(intX, intY, z); - } else { + tileCenter = [y, x, y + 0.1, x + 0.1]; + const { minX, minY } = new SphericalMercator().xyz(tileCenter, z); + xy = [minX, minY]; + } + const fetchTile = await fetchTileData(source, sourceType, z, x, y); + if (fetchTile == null) return res.status(204).send(); + + let data = fetchTile.data; + const image = new Image(); + await new Promise(async (resolve, reject) => { + image.onload = async () => { + const canvas = createCanvas(TILE_SIZE, TILE_SIZE); + const context = canvas.getContext('2d'); + context.drawImage(image, 0, 0); + const imgdata = context.getImageData(0, 0, TILE_SIZE, TILE_SIZE); + const arrayWidth = imgdata.width; + const arrayHeight = imgdata.height; + const bytesPerPixel = 4; + const xPixel = Math.floor(xy[0]); + const yPixel = Math.floor(xy[1]); if ( - z < tileJSON.minzoom || - z > tileJSON.maxzoom || - x < -180 || - y < -90 || - x > 180 || - y > 90 + xPixel < 0 || + yPixel < 0 || + xPixel >= arrayWidth || + yPixel >= arrayHeight ) { - return res.status(404).send('Out of bounds'); + return reject('Out of bounds Pixel'); } - tileCenter = [y, x, y + 0.1, x + 0.1]; - const { minX, minY } = new SphericalMercator().xyz(tileCenter, z); - xy = [minX, minY]; - } - const fetchTile = await fetchTileData( - source, - sourceType, - z, - x, - y, - ); - if (fetchTile == null) return res.status(204).send(); - - let data = fetchTile.data; - const image = new Image(); - await new Promise(async (resolve, reject) => { - image.onload = async () => { - const canvas = createCanvas(TILE_SIZE, TILE_SIZE); - const context = canvas.getContext('2d'); - context.drawImage(image, 0, 0); - const imgdata = context.getImageData(0, 0, TILE_SIZE, TILE_SIZE); - const arrayWidth = imgdata.width; - const arrayHeight = imgdata.height; - const bytesPerPixel = 4; - const xPixel = Math.floor(xy[0]); - const yPixel = Math.floor(xy[1]); - if ( - xPixel < 0 || - yPixel < 0 || - xPixel >= arrayWidth || - yPixel >= arrayHeight - ) { - return reject('Out of bounds Pixel'); - } - const index = (yPixel * arrayWidth + xPixel) * bytesPerPixel; - const red = imgdata.data[index]; - const green = imgdata.data[index + 1]; - const blue = imgdata.data[index + 2]; - let elevation; - if (encoding === 'mapbox') { - elevation = - -10000 + (red * 256 * 256 + green * 256 + blue) * 0.1; - } else if (encoding === 'terrarium') { - elevation = red * 256 + green + blue / 256 - 32768; - } else { - elevation = 'invalid encoding'; - } - resolve( - res.status(200).send({ - z, - x: xy[0], - y: xy[1], - red, - green, - blue, - latitude: tileCenter[0], - longitude: tileCenter[1], - elevation, - }), - ); - }; - image.onerror = (err) => reject(err); - if (format === 'webp') { - try { - const img = await sharp(data).toFormat('png').toBuffer(); - image.src = img; - } catch (err) { - reject(err); - } + const index = (yPixel * arrayWidth + xPixel) * bytesPerPixel; + const red = imgdata.data[index]; + const green = imgdata.data[index + 1]; + const blue = imgdata.data[index + 2]; + let elevation; + if (encoding === 'mapbox') { + elevation = -10000 + (red * 256 * 256 + green * 256 + blue) * 0.1; + } else if (encoding === 'terrarium') { + elevation = red * 256 + green + blue / 256 - 32768; } else { - image.src = data; + elevation = 'invalid encoding'; } - }); - } catch (err) { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } - }, - ); + resolve( + res.status(200).send({ + z, + x: xy[0], + y: xy[1], + red, + green, + blue, + latitude: tileCenter[0], + longitude: tileCenter[1], + elevation, + }), + ); + }; + image.onerror = (err) => reject(err); + if (format === 'webp') { + try { + const img = await sharp(data).toFormat('png').toBuffer(); + image.src = img; + } catch (err) { + reject(err); + } + } else { + image.src = data; + } + }); + } catch (err) { + return res + .status(500) + .header('Content-Type', 'text/plain') + .send(err.message); + } + }); app.get('/:id.json', (req, res) => { const item = repo[req.params.id]; From 0d72d5796a1f04933bd3f7b33733a121b66f6869 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Thu, 2 Jan 2025 22:08:12 -0500 Subject: [PATCH 041/104] add verbose logging, improve headers --- src/serve_data.js | 58 +++++++++++++++++++++++++---- src/serve_font.js | 21 ++++++++++- src/serve_light.js | 4 +- src/serve_rendered.js | 75 +++++++++++++++++++++++++++----------- src/serve_style.js | 7 +++- src/server.js | 85 ++++++++++++++++++++++++++++++++++++++----- 6 files changed, 207 insertions(+), 43 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 7d37f9018..a4379ff3a 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -17,11 +17,7 @@ import { isValidHttpUrl, fetchTileData, } from './utils.js'; -import { - getPMtilesInfo, - getPMtilesTile, - openPMtiles, -} from './pmtiles_adapter.js'; +import { getPMtilesInfo, openPMtiles } from './pmtiles_adapter.js'; import { gunzipP, gzipP } from './promises.js'; import { openMbTilesWrapper } from './mbtiles_wrapper.js'; @@ -30,12 +26,28 @@ export const serve_data = { * Initializes the serve_data module. * @param {object} options Configuration options. * @param {object} repo Repository object. + * @param {object} programOpts - An object containing the program options * @returns {express.Application} The initialized Express application. */ - init: function (options, repo) { + init: function (options, repo, programOpts) { + const { verbose } = programOpts; const app = express().disable('x-powered-by'); + /** + * Handles requests for tile data, responding with the tile image. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - ID of the tile. + * @param {string} req.params.z - Z coordinate of the tile. + * @param {string} req.params.x - X coordinate of the tile. + * @param {string} req.params.y - Y coordinate of the tile. + * @param {string} req.params.format - Format of the tile. + * @returns {Promise} + */ app.get('/:id/:z/:x/:y.:format', async (req, res) => { + if (verbose) { + console.log(req.params); + } const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); @@ -88,7 +100,14 @@ export const serve_data = { data = await gunzipP(data); isGzipped = false; } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + data = options.dataDecoratorFunc( + req.params.id, + 'data', + data, + z, + x, + y, + ); } } @@ -112,7 +131,9 @@ export const serve_data = { } data = JSON.stringify(geojson); } - delete headers['ETag']; // do not trust the tile ETag -- regenerate + if (headers) { + delete headers['ETag']; + } headers['Content-Encoding'] = 'gzip'; res.set(headers); @@ -123,6 +144,16 @@ export const serve_data = { return res.status(200).send(data); }); + /** + * Handles requests for elevation data. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - ID of the elevation data. + * @param {string} req.params.z - Z coordinate of the tile. + * @param {string} req.params.x - X coordinate of the tile (either integer or float). + * @param {string} req.params.y - Y coordinate of the tile (either integer or float). + * @returns {Promise} + */ app.get('/:id/elevation/:z/:x/:y', async (req, res, next) => { try { const item = repo?.[req.params.id]; @@ -189,6 +220,7 @@ export const serve_data = { const { minX, minY } = new SphericalMercator().xyz(tileCenter, z); xy = [minX, minY]; } + const fetchTile = await fetchTileData(source, sourceType, z, x, y); if (fetchTile == null) return res.status(204).send(); @@ -259,6 +291,13 @@ export const serve_data = { } }); + /** + * Handles requests for metadata for the tiles. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - ID of the data source. + * @returns {Promise} + */ app.get('/:id.json', (req, res) => { const item = repo[req.params.id]; if (!item) { @@ -289,6 +328,9 @@ export const serve_data = { * @param {object} params Parameters object. * @param {string} id ID of the data source. * @param {object} programOpts - An object containing the program options + * @param {string} programOpts.publicUrl Public URL for the data. + * @param {boolean} programOpts.verbose Whether verbose logging should be used. + * @param {Function} dataResolver Function to resolve data. * @returns {Promise} */ add: async function (options, repo, params, id, programOpts) { diff --git a/src/serve_font.js b/src/serve_font.js index 30f1fc8dc..49e6a41fa 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -8,9 +8,11 @@ import { getFontsPbf, listFonts } from './utils.js'; * Initializes and returns an Express app that serves font files. * @param {object} options - Configuration options for the server. * @param {object} allowedFonts - An object containing allowed fonts. + * @param {object} programOpts - An object containing the program options. * @returns {Promise} - A promise that resolves to the Express app. */ -export async function serve_font(options, allowedFonts) { +export async function serve_font(options, allowedFonts, programOpts) { + const { verbose } = programOpts; const app = express().disable('x-powered-by'); const lastModified = new Date().toUTCString(); @@ -19,7 +21,18 @@ export async function serve_font(options, allowedFonts) { const existingFonts = {}; + /** + * Handles requests for a font file. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.fontstack - Name of the font stack. + * @param {string} req.params.range - The range of the font (e.g. 0-255). + * @returns {Promise} + */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + if (verbose) { + console.log(req.params); + } const fontstack = decodeURI(req.params.fontstack); const range = req.params.range; @@ -41,6 +54,12 @@ export async function serve_font(options, allowedFonts) { } }); + /** + * Handles requests for a list of all available fonts. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @returns {void} + */ app.get('/fonts.json', (req, res) => { res.header('Content-type', 'application/json'); return res.send( diff --git a/src/serve_light.js b/src/serve_light.js index 474a78111..7e49c4929 100644 --- a/src/serve_light.js +++ b/src/serve_light.js @@ -3,7 +3,7 @@ 'use strict'; export const serve_rendered = { - init: (options, repo) => {}, - add: (options, repo, params, id, publicUrl, dataResolver) => {}, + init: (options, repo, programOpts) => {}, + add: (options, repo, params, id, programOpts, dataResolver) => {}, remove: (repo, id) => {}, }; diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 1f298b1a3..3413ca975 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -267,7 +267,6 @@ function extractPathsFromQuery(query, transformer) { } return paths; } - /** * Parses marker options provided via query and sets corresponding attributes * on marker object. @@ -626,6 +625,13 @@ const respondImage = async ( * @param {object} options - Configuration options for the server. * @param {object} repo - The repository object holding style data. * @param {object} req - Express request object. + * @param {string} req.params.id - The id of the style. + * @param {string} req.params.p1 - The tile size parameter, if available. + * @param {string} req.params.p2 - The z parameter. + * @param {string} req.params.p3 - The x parameter. + * @param {string} req.params.p4 - The y parameter. + * @param {string} req.params.scale - The scale parameter. + * @param {string} req.params.format - The format of the image. * @param {object} res - Express response object. * @param {Function} next - Express next middleware function. * @param {number} maxScaleFactor - The maximum scale factor allowed. @@ -641,12 +647,12 @@ async function handleTileRequest( ) { const { id, + p1: tileSize, p2: zParam, p3: xParam, p4: yParam, scale: scaleParam, format, - p1: tileSize, } = req.params; const item = repo[id]; if (!item) { @@ -694,6 +700,12 @@ async function handleTileRequest( * @param {object} repo - The repository object holding style data. * @param {object} req - Express request object. * @param {object} res - Express response object. + * @param {string} req.params.p2 - The raw or static parameter. + * @param {string} req.params.p3 - The staticType parameter. + * @param {string} req.params.p4 - The width parameter. + * @param {string} req.params.p5 - The height parameter. + * @param {string} req.params.scale - The scale parameter. + * @param {string} req.params.format - The format of the image. * @param {Function} next - Express next middleware function. * @param {number} maxScaleFactor - The maximum scale factor allowed. * @returns {Promise} @@ -708,32 +720,24 @@ async function handleStaticRequest( ) { const { id, - scale: scaleParam, - format, p2: raw, - p3: type, + p3: staticType, p4: width, p5: height, + scale: scaleParam, + format, } = req.params; const item = repo[id]; const parsedWidth = parseInt(width) || 512; const parsedHeight = parseInt(height) || 512; const scale = parseScale(scaleParam, maxScaleFactor); - let isRaw = raw !== undefined; - let staticType = type; - - if (!staticType) { - //workaround for type when raw is not set - staticType = raw; - isRaw = false; - } + let isRaw = raw === 'raw'; if (!item || !staticType || !format || !scale) { return res.sendStatus(404); } const staticTypeMatch = staticType.match(staticTypeRegex); - console.log(staticTypeMatch); if (staticTypeMatch.groups.lon) { // Center Based Static Image const z = parseFloat(staticTypeMatch.groups.zoom) || 0; @@ -765,7 +769,7 @@ async function handleStaticRequest( // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', ); } else if (staticTypeMatch.groups.minx) { // Area Based Static Image @@ -800,12 +804,12 @@ async function handleStaticRequest( const markers = extractMarkersFromQuery(req.query, options, transformer); // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', ); } else if (staticTypeMatch.groups.auto) { // Area Static Image @@ -859,12 +863,12 @@ async function handleStaticRequest( // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', ); } else { return res.sendStatus(404); @@ -879,18 +883,37 @@ export const serve_rendered = { * Initializes the serve_rendered module. * @param {object} options Configuration options. * @param {object} repo Repository object. + * @param {object} programOpts - An object containing the program options. * @returns {Promise} A promise that resolves to the Express app. */ - init: async function (options, repo) { + init: async function (options, repo, programOpts) { + const { verbose } = programOpts; maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); const app = express().disable('x-powered-by'); + /** + * Handles requests for tile images. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - The id of the style. + * @param {string} req.params.p1 - The tile size or static parameter, if available + * @param {string} req.params.p2 - The z, static, or raw parameter. + * @param {string} req.params.p3 - The x or staticType parameter. + * @param {string} req.params.p4 - The y or width parameter. + * @param {string} req.params.p5 - The height parameter. + * @param {string} req.params.scale - The scale parameter. + * @param {string} req.params.format - The format of the image. + * @returns {Promise} + */ app.get( `/:id{/:p1}/:p2/:p3/:p4{x:p5}{@:scale}{.:format}`, async (req, res, next) => { try { - const { p2 } = req.params; - if (p2 === 'static') { + if (verbose) { + console.log(req.params); + } + const { p1, p2 } = req.params; + if ((!p1 && p2 === 'static') || (p1 === 'static' && p2 === 'raw')) { // Route to static if p2 is static if (options.serveStaticMaps !== false) { return handleStaticRequest( @@ -920,6 +943,14 @@ export const serve_rendered = { }, ); + /** + * Handles requests for tile json endpoint. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - The id of the tilejson + * @param {string} [req.params.tileSize] - The size of the tile, if specified. + * @returns {void} + */ app.get('{/:tileSize}/:id.json', (req, res, next) => { const item = repo[req.params.id]; if (!item) { diff --git a/src/serve_style.js b/src/serve_style.js index 5a7478288..b5f59aac5 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -29,9 +29,11 @@ export const serve_style = { * Initializes the serve_style module. * @param {object} options Configuration options. * @param {object} repo Repository object. + * @param {object} programOpts - An object containing the program options. * @returns {express.Application} The initialized Express application. */ - init: function (options, repo) { + init: function (options, repo, programOpts) { + const { verbose } = programOpts; const app = express().disable('x-powered-by'); app.get('/:id/style.json', (req, res, next) => { @@ -64,6 +66,9 @@ export const serve_style = { }); app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + if (verbose) { + console.log(req.params); + } const { spriteID = 'default', id, format } = req.params; const spriteScale = allowedSpriteScales(req.params.scale); diff --git a/src/server.js b/src/server.js index beee4b36f..524d1ae2b 100644 --- a/src/server.js +++ b/src/server.js @@ -76,7 +76,7 @@ async function start(opts) { config = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch (e) { console.log('ERROR: Config file not found or invalid!'); - console.log(' See README.md for instructions and sample data.'); + console.log(' See README.md for instructions and sample data.'); process.exit(1); } } @@ -167,12 +167,12 @@ async function start(opts) { app.use(cors()); } - app.use('/data/', serve_data.init(options, serving.data)); + app.use('/data/', serve_data.init(options, serving.data, opts)); app.use('/files/', express.static(paths.files)); - app.use('/styles/', serve_style.init(options, serving.styles)); + app.use('/styles/', serve_style.init(options, serving.styles, opts)); if (!isLight) { startupPromises.push( - serve_rendered.init(options, serving.rendered).then((sub) => { + serve_rendered.init(options, serving.rendered, opts).then((sub) => { app.use('/styles/', sub); }), ); @@ -288,7 +288,7 @@ async function start(opts) { addStyle(id, item, true, true); } startupPromises.push( - serve_font(options, serving.fonts).then((sub) => { + serve_font(options, serving.fonts, opts).then((sub) => { app.use('/', sub); }), ); @@ -342,6 +342,13 @@ async function start(opts) { } }); } + /** + * Handles requests for a list of available styles. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} [req.query.key] - Optional API key. + * @returns {void} + */ app.get('/styles.json', (req, res, next) => { const result = []; const query = req.query.key @@ -395,15 +402,35 @@ async function start(opts) { return arr; } + /** + * Handles requests for a rendered tilejson endpoint. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.tileSize - Optional tile size parameter. + * @returns {void} + */ app.get('{/:tileSize}/rendered.json', (req, res, next) => { const tileSize = allowedTileSizes(req.params['tileSize']); res.send(addTileJSONs([], req, 'rendered', parseInt(tileSize, 10))); }); + /** + * Handles requests for a data tilejson endpoint. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @returns {void} + */ app.get('/data.json', (req, res) => { res.send(addTileJSONs([], req, 'data', undefined)); }); + /** + * Handles requests for a combined rendered and data tilejson endpoint. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.tileSize - Optional tile size parameter. + * @returns {void} + */ app.get('{/:tileSize}/index.json', (req, res, next) => { const tileSize = allowedTileSizes(req.params['tileSize']); res.send( @@ -421,6 +448,7 @@ async function start(opts) { app.use('/', express.static(path.join(__dirname, '../public/resources'))); const templates = path.join(__dirname, '../public/templates'); + /** * Serves a Handlebars template. * @param {string} urlPath - The URL path to serve the template at @@ -477,6 +505,12 @@ async function start(opts) { } } + /** + * Handles requests for the index page, providing a list of available styles and data. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @returns {void} + */ serveTemplate('/', 'index', (req) => { let styles = {}; for (const id of Object.keys(serving.styles || {})) { @@ -489,11 +523,15 @@ async function start(opts) { if (style.serving_rendered) { const { center } = style.serving_rendered.tileJSON; if (center) { - style.viewer_hash = `#${center[2]}/${center[1].toFixed(5)}/${center[0].toFixed(5)}`; + style.viewer_hash = `#${center[2]}/${center[1].toFixed( + 5, + )}/${center[0].toFixed(5)}`; const centerPx = mercator.px([center[0], center[1]], center[2]); // Set thumbnail default size to be 256px x 256px - style.thumbnail = `${Math.floor(center[2])}/${Math.floor(centerPx[0] / 256)}/${Math.floor(centerPx[1] / 256)}.png`; + style.thumbnail = `${Math.floor(center[2])}/${Math.floor( + centerPx[0] / 256, + )}/${Math.floor(centerPx[1] / 256)}.png`; } const tileSize = 512; @@ -549,7 +587,9 @@ async function start(opts) { } if (center) { const centerPx = mercator.px([center[0], center[1]], center[2]); - data.thumbnail = `${Math.floor(center[2])}/${Math.floor(centerPx[0] / 256)}/${Math.floor(centerPx[1] / 256)}.${tileJSON.format}`; + data.thumbnail = `${Math.floor(center[2])}/${Math.floor( + centerPx[0] / 256, + )}/${Math.floor(centerPx[1] / 256)}.${tileJSON.format}`; } } @@ -574,6 +614,13 @@ async function start(opts) { }; }); + /** + * Handles requests for a map viewer template for a specific style. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - ID of the style. + * @returns {void} + */ serveTemplate('/styles/:id/', 'viewer', (req) => { const { id } = req.params; const style = clone(((serving.styles || {})[id] || {}).styleJSON); @@ -590,6 +637,13 @@ async function start(opts) { }; }); + /** + * Handles requests for a Web Map Tile Service (WMTS) XML template. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - ID of the style. + * @returns {void} + */ serveTemplate('/styles/:id/wmts.xml', 'wmts', (req) => { const { id } = req.params; const wmts = clone((serving.styles || {})[id]); @@ -621,6 +675,14 @@ async function start(opts) { }; }); + /** + * Handles requests for a data view template for a specific data source. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.id - ID of the data source. + * @param {string} [req.params.view] - Optional view type. + * @returns {void} + */ serveTemplate('/data{/:view}/:id/', 'data', (req) => { const { id, view } = req.params; const data = serving.data[id]; @@ -649,6 +711,12 @@ async function start(opts) { startupComplete = true; }); + /** + * Handles requests to see the health of the server. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @returns {void} + */ app.get('/health', (req, res) => { if (startupComplete) { return res.status(200).send('OK'); @@ -678,7 +746,6 @@ async function start(opts) { startupPromise, }; } - /** * Stop the server gracefully * @param {string} signal Name of the received signal From 468779b1ddd455d3ee4f5b52e1a44882b270fae4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Thu, 2 Jan 2025 22:18:51 -0500 Subject: [PATCH 042/104] try to fix codeql Information exposure through a stack trace --- src/serve_font.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 49e6a41fa..0319e9049 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -23,6 +23,7 @@ export async function serve_font(options, allowedFonts, programOpts) { /** * Handles requests for a font file. + * * @param {object} req - Express request object. * @param {object} res - Express response object. * @param {string} req.params.fontstack - Name of the font stack. @@ -49,13 +50,17 @@ export async function serve_font(options, allowedFonts, programOpts) { res.header('Last-Modified', lastModified); return res.send(concatenated); } catch (err) { - console.error('Error serving font:', err); - return res.status(400).header('Content-Type', 'text/plain').send(err); + console.error(`Error serving font: ${fontstack}/${range}.pbf`, err); + return res + .status(400) + .header('Content-Type', 'text/plain') + .send('Error serving font'); } }); /** * Handles requests for a list of all available fonts. + * * @param {object} req - Express request object. * @param {object} res - Express response object. * @returns {void} From 8bdd14a5e9c871581e861bceca4d08238e07aba7 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 01:53:32 -0500 Subject: [PATCH 043/104] test --- src/serve_data.js | 16 +++++- src/serve_font.js | 9 ++- src/serve_rendered.js | 124 +++++++++++++++++++++++++++++++---------- src/serve_style.js | 98 ++++++++++++++++++++++---------- test/static.js | 2 +- test/tiles_rendered.js | 14 ++--- 6 files changed, 191 insertions(+), 72 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index a4379ff3a..685f18d82 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -46,7 +46,9 @@ export const serve_data = { */ app.get('/:id/:z/:x/:y.:format', async (req, res) => { if (verbose) { - console.log(req.params); + console.log( + `Handling tile request for: /data/${req.params.id}/${req.params.z}/${req.params.x}/${req.params.y}.${req.params.format}`, + ); } const item = repo[req.params.id]; if (!item) { @@ -156,6 +158,11 @@ export const serve_data = { */ app.get('/:id/elevation/:z/:x/:y', async (req, res, next) => { try { + if (verbose) { + console.log( + `Handling elevation request for: /data/${req.params.id}/elevation/${req.params.z}/${req.params.x}/${req.params.y}`, + ); + } const item = repo?.[req.params.id]; if (!item) return res.sendStatus(404); if (!item.source) return res.status(404).send('Missing source'); @@ -292,13 +299,18 @@ export const serve_data = { }); /** - * Handles requests for metadata for the tiles. + * Handles requests for tilejson for the data tiles. * @param {object} req - Express request object. * @param {object} res - Express response object. * @param {string} req.params.id - ID of the data source. * @returns {Promise} */ app.get('/:id.json', (req, res) => { + if (verbose) { + console.log( + `Handling tilejson request for: /data/${req.params.id}.json`, + ); + } const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); diff --git a/src/serve_font.js b/src/serve_font.js index 0319e9049..020d6d585 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -23,7 +23,6 @@ export async function serve_font(options, allowedFonts, programOpts) { /** * Handles requests for a font file. - * * @param {object} req - Express request object. * @param {object} res - Express response object. * @param {string} req.params.fontstack - Name of the font stack. @@ -32,7 +31,9 @@ export async function serve_font(options, allowedFonts, programOpts) { */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { if (verbose) { - console.log(req.params); + console.log( + `Handling font request for: /fonts/${req.params.fontstack}/${req.params.range}.pbf`, + ); } const fontstack = decodeURI(req.params.fontstack); const range = req.params.range; @@ -60,12 +61,14 @@ export async function serve_font(options, allowedFonts, programOpts) { /** * Handles requests for a list of all available fonts. - * * @param {object} req - Express request object. * @param {object} res - Express response object. * @returns {void} */ app.get('/fonts.json', (req, res) => { + if (verbose) { + console.log('Handling list font request for /fonts.json'); + } res.header('Content-type', 'application/json'); return res.send( Object.keys(options.serveAllFonts ? existingFonts : allowedFonts).sort(), diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 3413ca975..3a4563984 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -34,6 +34,7 @@ import { isValidHttpUrl, fixTileJSONCenter, fetchTileData, + allowedOptions, } from './utils.js'; import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; import { renderOverlay, renderWatermark, renderAttribution } from './render.js'; @@ -635,6 +636,7 @@ const respondImage = async ( * @param {object} res - Express response object. * @param {Function} next - Express next middleware function. * @param {number} maxScaleFactor - The maximum scale factor allowed. + * @param defailtTileSize * @returns {Promise} */ async function handleTileRequest( @@ -644,6 +646,7 @@ async function handleTileRequest( res, next, maxScaleFactor, + defailtTileSize, ) { const { id, @@ -658,6 +661,7 @@ async function handleTileRequest( if (!item) { return res.sendStatus(404); } + console.log(req.params); const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); @@ -670,7 +674,19 @@ async function handleTileRequest( const x = parseFloat(xParam) | 0; const y = parseFloat(yParam) | 0; const scale = parseScale(scaleParam, maxScaleFactor); - const parsedTileSize = parseInt(tileSize, 10) || 256; + + let parsedTileSize = defailtTileSize; + if (tileSize) { + const allowedTileSizes = allowedOptions(['256', '512'], { + defaultValue: null, + }); + parsedTileSize = allowedTileSizes(tileSize); + + if (parsedTileSize == null) { + return res.status(400).send('Invalid Tile Size'); + } + } + if ( scale == null || z < 0 || @@ -680,7 +696,7 @@ async function handleTileRequest( x >= Math.pow(2, z) || y >= Math.pow(2, z) ) { - return res.status(404).send('Out of bounds'); + return res.status(400).send('Out of bounds'); } const tileCenter = mercator.ll( @@ -722,14 +738,43 @@ async function handleStaticRequest( id, p2: raw, p3: staticType, - p4: width, - p5: height, + p4: widthAndHeight, scale: scaleParam, format, } = req.params; + console.log(req.params); const item = repo[id]; - const parsedWidth = parseInt(width) || 512; - const parsedHeight = parseInt(height) || 512; + + let parsedWidth = null; + let parsedHeight = null; + if (widthAndHeight) { + const sizeMatch = widthAndHeight.match(/^(\d+)x(\d+)$/); + if (sizeMatch) { + const width = parseInt(sizeMatch[1], 10); + const height = parseInt(sizeMatch[2], 10); + + if ( + isNaN(width) || + isNaN(height) || + width !== parseFloat(sizeMatch[1]) || + height !== parseFloat(sizeMatch[2]) + ) { + return res + .status(400) + .send('Invalid width or height provided in size parameter'); + } + parsedWidth = width; + parsedHeight = height; + } else { + return res + .status(400) + .send('Invalid width or height provided in size parameter'); + } + } else { + return res + .status(400) + .send('Invalid width or height provided in size parameter'); + } const scale = parseScale(scaleParam, maxScaleFactor); let isRaw = raw === 'raw'; @@ -740,11 +785,12 @@ async function handleStaticRequest( const staticTypeMatch = staticType.match(staticTypeRegex); if (staticTypeMatch.groups.lon) { // Center Based Static Image - const z = parseFloat(staticTypeMatch.groups.zoom) || 0; - let x = parseFloat(staticTypeMatch.groups.lon) || 0; - let y = parseFloat(staticTypeMatch.groups.lat) || 0; - const bearing = parseFloat(staticTypeMatch.groups.bearing) || 0; - const pitch = parseInt(staticTypeMatch.groups.pitch) || 0; + const z = staticTypeMatch.groups.zoom; + let x = staticTypeMatch.groups.lon; + let y = staticTypeMatch.groups.lat; + const bearing = staticTypeMatch.groups.bearing; + const pitch = staticTypeMatch.groups.pitch; + if (z < 0) { return res.status(404).send('Invalid zoom'); } @@ -764,13 +810,13 @@ async function handleStaticRequest( // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, - ); + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', - ); + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); } else if (staticTypeMatch.groups.minx) { // Area Based Static Image const bbox = [ @@ -802,15 +848,16 @@ async function handleStaticRequest( const pitch = 0; const paths = extractPathsFromQuery(req.query, transformer); const markers = extractMarkersFromQuery(req.query, options, transformer); + // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, - ); + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', - ); + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); } else if (staticTypeMatch.groups.auto) { // Area Static Image const bearing = 0; @@ -863,13 +910,13 @@ async function handleStaticRequest( // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, - ); + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', - ); + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); } else { return res.sendStatus(404); } @@ -887,7 +934,7 @@ export const serve_rendered = { * @returns {Promise} A promise that resolves to the Express app. */ init: async function (options, repo, programOpts) { - const { verbose } = programOpts; + const { verbose, tileSize: defailtTileSize = 256 } = programOpts; maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); const app = express().disable('x-powered-by'); @@ -896,7 +943,7 @@ export const serve_rendered = { * @param {object} req - Express request object. * @param {object} res - Express response object. * @param {string} req.params.id - The id of the style. - * @param {string} req.params.p1 - The tile size or static parameter, if available + * @param {string} [req.params.p1] - The tile size or static parameter, if available. * @param {string} req.params.p2 - The z, static, or raw parameter. * @param {string} req.params.p3 - The x or staticType parameter. * @param {string} req.params.p4 - The y or width parameter. @@ -906,14 +953,24 @@ export const serve_rendered = { * @returns {Promise} */ app.get( - `/:id{/:p1}/:p2/:p3/:p4{x:p5}{@:scale}{.:format}`, + `/:id{/:p1}/:p2/:p3/:p4{@:scale}{.:format}`, async (req, res, next) => { try { + const { p1, p2, id, p3, p4, p5, scale, format } = req.params; + const requestType = + (!p1 && p2 === 'static') || (p1 === 'static' && p2 === 'raw') + ? 'static' + : 'tile'; + if (verbose) { - console.log(req.params); + console.log( + `Handling rendered ${requestType} request for: /styles/${id}${p1 ? '/' + p1 : ''}/${p2}/${p3}/${p4}${p5 ? 'x' + p5 : ''}${ + scale ? '@' + scale : '' + }.${format}`, + ); } - const { p1, p2 } = req.params; - if ((!p1 && p2 === 'static') || (p1 === 'static' && p2 === 'raw')) { + + if (requestType === 'static') { // Route to static if p2 is static if (options.serveStaticMaps !== false) { return handleStaticRequest( @@ -923,6 +980,7 @@ export const serve_rendered = { res, next, maxScaleFactor, + defailtTileSize, ); } return res.sendStatus(404); @@ -935,6 +993,7 @@ export const serve_rendered = { res, next, maxScaleFactor, + defailtTileSize, ); } catch (e) { console.log(e); @@ -944,7 +1003,7 @@ export const serve_rendered = { ); /** - * Handles requests for tile json endpoint. + * Handles requests for rendered tilejson endpoint. * @param {object} req - Express request object. * @param {object} res - Express response object. * @param {string} req.params.id - The id of the tilejson @@ -957,6 +1016,11 @@ export const serve_rendered = { return res.sendStatus(404); } const tileSize = parseInt(req.params.tileSize, 10) || undefined; + if (verbose) { + console.log( + `Handling rendered tilejson request for: /styles/${tileSize ? tileSize + '/' : ''}${req.params.id}.json`, + ); + } const info = clone(item.tileJSON); info.tiles = getTileUrls( req, diff --git a/src/serve_style.js b/src/serve_style.js index b5f59aac5..80f42e6f3 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -35,59 +35,95 @@ export const serve_style = { init: function (options, repo, programOpts) { const { verbose } = programOpts; const app = express().disable('x-powered-by'); - + /** + * Handles requests for style.json files. + * @param {express.Request} req - Express request object. + * @param {express.Response} res - Express response object. + * @param {express.NextFunction} next - Express next function. + * @param {string} req.params.id - ID of the style. + * @returns {Promise} + */ app.get('/:id/style.json', (req, res, next) => { - const item = repo[req.params.id]; - if (!item) { - return res.sendStatus(404); + const { id } = req.params; + if (verbose) { + console.log(`Handling style request for: /styles/${id}/style.json`); } - const styleJSON_ = clone(item.styleJSON); - for (const name of Object.keys(styleJSON_.sources)) { - const source = styleJSON_.sources[name]; - source.url = fixUrl(req, source.url, item.publicUrl); - if (typeof source.data == 'string') { - source.data = fixUrl(req, source.data, item.publicUrl); + try { + const item = repo[id]; + if (!item) { + return res.sendStatus(404); } - } - // mapbox-gl-js viewer cannot handle sprite urls with query - if (styleJSON_.sprite) { - if (Array.isArray(styleJSON_.sprite)) { - styleJSON_.sprite.forEach((spriteItem) => { - spriteItem.url = fixUrl(req, spriteItem.url, item.publicUrl); - }); - } else { - styleJSON_.sprite = fixUrl(req, styleJSON_.sprite, item.publicUrl); + const styleJSON_ = clone(item.styleJSON); + for (const name of Object.keys(styleJSON_.sources)) { + const source = styleJSON_.sources[name]; + source.url = fixUrl(req, source.url, item.publicUrl); + if (typeof source.data == 'string') { + source.data = fixUrl(req, source.data, item.publicUrl); + } } + if (styleJSON_.sprite) { + if (Array.isArray(styleJSON_.sprite)) { + styleJSON_.sprite.forEach((spriteItem) => { + spriteItem.url = fixUrl(req, spriteItem.url, item.publicUrl); + }); + } else { + styleJSON_.sprite = fixUrl(req, styleJSON_.sprite, item.publicUrl); + } + } + if (styleJSON_.glyphs) { + styleJSON_.glyphs = fixUrl(req, styleJSON_.glyphs, item.publicUrl); + } + return res.send(styleJSON_); + } catch (e) { + next(e); } - if (styleJSON_.glyphs) { - styleJSON_.glyphs = fixUrl(req, styleJSON_.glyphs, item.publicUrl); - } - return res.send(styleJSON_); }); + /** + * Handles GET requests for sprite images and JSON files. + * @param {express.Request} req - Express request object. + * @param {express.Response} res - Express response object. + * @param {express.NextFunction} next - Express next function. + * @param {string} req.params.id - ID of the sprite. + * @param {string} [req.params.spriteID='default'] - ID of the specific sprite image, defaults to 'default'. + * @param {string} [req.params.scale] - Scale of the sprite image, defaults to ''. + * @param {string} req.params.format - Format of the sprite file, 'png' or 'json'. + * @returns {Promise} + */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + const { spriteID = 'default', id, format, scale } = req.params; + const spriteScale = allowedSpriteScales(scale); + if (verbose) { - console.log(req.params); + console.log( + `Handling sprite request for: /${id}/sprite/${spriteID}${scale}.${format}`, + ); } - const { spriteID = 'default', id, format } = req.params; - const spriteScale = allowedSpriteScales(req.params.scale); - const item = repo[id]; if (!item || !allowedSpriteFormats(format)) { + if (verbose) + console.error( + `Sprite item or format not found for: /${id}/sprite/${spriteID}${scale}.${format}`, + ); return res.sendStatus(404); } - const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); if (!sprite) { + if (verbose) + console.error( + `Sprite not found for: /${id}/sprite/${spriteID}${scale}.${format}`, + ); return res.status(400).send('Bad Sprite ID or Scale'); } const filename = `${sprite.path}${spriteScale}.${format}`; + if (verbose) console.log(`Loading sprite from: ${filename}`); // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { - console.error('Sprite load error: %s, Error: %s', filename, err); + if (verbose) + console.error('Sprite load error: %s, Error: %s', filename, err); return res.sendStatus(404); } @@ -96,6 +132,10 @@ export const serve_style = { } else if (format === 'png') { res.header('Content-type', 'image/png'); } + if (verbose) + console.log( + `Responding with sprite data for /${id}/sprite/${spriteID}${scale}.${format}`, + ); return res.send(data); }); }); diff --git a/test/static.js b/test/static.js index 32bd80c77..e6183bf7f 100644 --- a/test/static.js +++ b/test/static.js @@ -135,7 +135,7 @@ describe('Static endpoints', function () { testStatic(prefix, '0,0,1,1/1x1', 'gif', 400); - testStatic(prefix, '-180,-80,180,80/0.5x2.6', 'png', 404); + testStatic(prefix, '-180,-80,180,80/0.5x2.6', 'png', 400); }); }); diff --git a/test/tiles_rendered.js b/test/tiles_rendered.js index 6f7f43809..61996d93c 100644 --- a/test/tiles_rendered.js +++ b/test/tiles_rendered.js @@ -60,16 +60,16 @@ describe('Raster tiles', function () { describe('invalid requests return 4xx', function () { testTile('non_existent', 256, 0, 0, 0, 'png', 404); - testTile(prefix, 256, -1, 0, 0, 'png', 404); - testTile(prefix, 256, 25, 0, 0, 'png', 404); - testTile(prefix, 256, 0, 1, 0, 'png', 404); - testTile(prefix, 256, 0, 0, 1, 'png', 404); + testTile(prefix, 256, -1, 0, 0, 'png', 400); + testTile(prefix, 256, 25, 0, 0, 'png', 400); + testTile(prefix, 256, 0, 1, 0, 'png', 400); + testTile(prefix, 256, 0, 0, 1, 'png', 400); testTile(prefix, 256, 0, 0, 0, 'gif', 400); testTile(prefix, 256, 0, 0, 0, 'pbf', 400); - testTile(prefix, 256, 0, 0, 0, 'png', 404, 1); - testTile(prefix, 256, 0, 0, 0, 'png', 404, 5); + testTile(prefix, 256, 0, 0, 0, 'png', 400, 1); + testTile(prefix, 256, 0, 0, 0, 'png', 400, 5); - testTile(prefix, 300, 0, 0, 0, 'png', 404); + testTile(prefix, 300, 0, 0, 0, 'png', 400); }); }); From 32606113c0d793b82d19c160be75e3be69357ae3 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 03:48:17 -0500 Subject: [PATCH 044/104] all tests passing --- src/serve_rendered.js | 76 ++++++++++++++++++++++++++----------------- test/static.js | 2 +- 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 3a4563984..e76c41e75 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -453,6 +453,8 @@ const respondImage = async ( overlay = null, mode = 'tile', ) => { + console.log(lat); + console.log(lon); if ( Math.abs(lon) > 180 || Math.abs(lat) > 85.06 || @@ -724,6 +726,7 @@ async function handleTileRequest( * @param {string} req.params.format - The format of the image. * @param {Function} next - Express next middleware function. * @param {number} maxScaleFactor - The maximum scale factor allowed. + * @param verbose * @returns {Promise} */ async function handleStaticRequest( @@ -733,6 +736,7 @@ async function handleStaticRequest( res, next, maxScaleFactor, + verbose, ) { const { id, @@ -742,6 +746,13 @@ async function handleStaticRequest( scale: scaleParam, format, } = req.params; + if (verbose) { + console.log( + `Handling static request for: /styles/${id}/static/${raw ? raw + '/' : ''}${staticType}${widthAndHeight ? '/' + widthAndHeight : ''}${ + scaleParam ? '@' + scaleParam : '' + }.${format}`, + ); + } console.log(req.params); const item = repo[id]; @@ -752,7 +763,6 @@ async function handleStaticRequest( if (sizeMatch) { const width = parseInt(sizeMatch[1], 10); const height = parseInt(sizeMatch[2], 10); - if ( isNaN(width) || isNaN(height) || @@ -775,22 +785,22 @@ async function handleStaticRequest( .status(400) .send('Invalid width or height provided in size parameter'); } + const scale = parseScale(scaleParam, maxScaleFactor); let isRaw = raw === 'raw'; - if (!item || !staticType || !format || !scale) { + const staticTypeMatch = staticType.match(staticTypeRegex); + if (!item || !format || !scale || !staticTypeMatch?.groups) { return res.sendStatus(404); } - const staticTypeMatch = staticType.match(staticTypeRegex); if (staticTypeMatch.groups.lon) { // Center Based Static Image - const z = staticTypeMatch.groups.zoom; - let x = staticTypeMatch.groups.lon; - let y = staticTypeMatch.groups.lat; - const bearing = staticTypeMatch.groups.bearing; - const pitch = staticTypeMatch.groups.pitch; - + const z = parseFloat(staticTypeMatch.groups.zoom) || 0; + let x = parseFloat(staticTypeMatch.groups.lon) || 0; + let y = parseFloat(staticTypeMatch.groups.lat) || 0; + const bearing = parseFloat(staticTypeMatch.groups.bearing) || 0; + const pitch = parseInt(staticTypeMatch.groups.pitch) || 0; if (z < 0) { return res.status(404).send('Invalid zoom'); } @@ -807,24 +817,27 @@ async function handleStaticRequest( const paths = extractPathsFromQuery(req.query, transformer); const markers = extractMarkersFromQuery(req.query, options, transformer); - // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, - ); + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', - ); + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); } else if (staticTypeMatch.groups.minx) { // Area Based Static Image - const bbox = [ - +staticTypeMatch.groups.minx, - +staticTypeMatch.groups.miny, - +staticTypeMatch.groups.maxx, - +staticTypeMatch.groups.maxx, - ]; + const minx = parseFloat(staticTypeMatch.groups.minx) || 0; + const miny = parseFloat(staticTypeMatch.groups.miny) || 0; + const maxx = parseFloat(staticTypeMatch.groups.maxx) || 0; + const maxy = parseFloat(staticTypeMatch.groups.maxy) || 0; + if (isNaN(minx) || isNaN(miny) || isNaN(maxx) || isNaN(maxy)) { + return res + .status(400) + .send('Invalid bounding box provided in staticType parameter'); + } + const bbox = [minx, miny, maxx, maxy]; let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; const transformer = isRaw @@ -841,23 +854,27 @@ async function handleStaticRequest( center = transformer(center); } + if (Math.abs(center[0]) > 180 || Math.abs(center[1]) > 85.06) { + return res.status(400).send('Invalid center'); + } + const z = calcZForBBox(bbox, parsedWidth, parsedHeight, req.query); const x = center[0]; const y = center[1]; const bearing = 0; const pitch = 0; + const paths = extractPathsFromQuery(req.query, transformer); const markers = extractMarkersFromQuery(req.query, options, transformer); - // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', - ); + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); } else if (staticTypeMatch.groups.auto) { // Area Static Image const bearing = 0; @@ -910,18 +927,17 @@ async function handleStaticRequest( // prettier-ignore const overlay = await renderOverlay( - z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, - ); + z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, paths, markers, req.query, + ); // prettier-ignore return await respondImage( - options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', - ); + options, item, z, x, y, bearing, pitch, parsedWidth, parsedHeight, scale, format, res, overlay, 'static', + ); } else { return res.sendStatus(404); } } - const existingFonts = {}; let maxScaleFactor = 2; @@ -980,7 +996,7 @@ export const serve_rendered = { res, next, maxScaleFactor, - defailtTileSize, + verbose, ); } return res.sendStatus(404); diff --git a/test/static.js b/test/static.js index e6183bf7f..dedc793ec 100644 --- a/test/static.js +++ b/test/static.js @@ -78,7 +78,7 @@ describe('Static endpoints', function () { testStatic(prefix, '0,0,0/256x256', 'png', 404, 1); testStatic(prefix, '0,0,-1/256x256', 'png', 404); - testStatic(prefix, '0,0,0/256.5x256.5', 'png', 404); + testStatic(prefix, '0,0,0/256.5x256.5', 'png', 400); testStatic(prefix, '0,0,0,/256x256', 'png', 404); testStatic(prefix, '0,0,0,0,/256x256', 'png', 404); From ff4ab841594a569eb65eddd3b87111c7982454de Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 03:55:46 -0500 Subject: [PATCH 045/104] cleanup unneeded changes --- src/serve_rendered.js | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index e76c41e75..d41a0aef1 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -453,8 +453,6 @@ const respondImage = async ( overlay = null, mode = 'tile', ) => { - console.log(lat); - console.log(lon); if ( Math.abs(lon) > 180 || Math.abs(lat) > 85.06 || @@ -748,9 +746,7 @@ async function handleStaticRequest( } = req.params; if (verbose) { console.log( - `Handling static request for: /styles/${id}/static/${raw ? raw + '/' : ''}${staticType}${widthAndHeight ? '/' + widthAndHeight : ''}${ - scaleParam ? '@' + scaleParam : '' - }.${format}`, + `Handling static request for: /styles/${id}/static/${raw ? raw + '/' : ''}${staticType}${widthAndHeight ? '/' + widthAndHeight : ''}${scaleParam ? '@' + scaleParam : ''}.${format}`, ); } console.log(req.params); @@ -832,11 +828,6 @@ async function handleStaticRequest( const miny = parseFloat(staticTypeMatch.groups.miny) || 0; const maxx = parseFloat(staticTypeMatch.groups.maxx) || 0; const maxy = parseFloat(staticTypeMatch.groups.maxy) || 0; - if (isNaN(minx) || isNaN(miny) || isNaN(maxx) || isNaN(maxy)) { - return res - .status(400) - .send('Invalid bounding box provided in staticType parameter'); - } const bbox = [minx, miny, maxx, maxy]; let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; @@ -854,10 +845,6 @@ async function handleStaticRequest( center = transformer(center); } - if (Math.abs(center[0]) > 180 || Math.abs(center[1]) > 85.06) { - return res.status(400).send('Invalid center'); - } - const z = calcZForBBox(bbox, parsedWidth, parsedHeight, req.query); const x = center[0]; const y = center[1]; From d0aad8e8969847ad8d896498ed44832e88577c64 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 04:01:34 -0500 Subject: [PATCH 046/104] cleanup --- src/serve_rendered.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index d41a0aef1..e7055aedd 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -661,7 +661,6 @@ async function handleTileRequest( if (!item) { return res.sendStatus(404); } - console.log(req.params); const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); @@ -734,7 +733,6 @@ async function handleStaticRequest( res, next, maxScaleFactor, - verbose, ) { const { id, @@ -744,12 +742,6 @@ async function handleStaticRequest( scale: scaleParam, format, } = req.params; - if (verbose) { - console.log( - `Handling static request for: /styles/${id}/static/${raw ? raw + '/' : ''}${staticType}${widthAndHeight ? '/' + widthAndHeight : ''}${scaleParam ? '@' + scaleParam : ''}.${format}`, - ); - } - console.log(req.params); const item = repo[id]; let parsedWidth = null; @@ -983,7 +975,6 @@ export const serve_rendered = { res, next, maxScaleFactor, - verbose, ); } return res.sendStatus(404); From 4c58ebb7858dcdd1febc93530fc88958953d97a8 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 12:11:41 -0500 Subject: [PATCH 047/104] try to fix codeql error --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 9a506c9b9..d9133a805 100644 --- a/src/utils.js +++ b/src/utils.js @@ -198,7 +198,7 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return new Promise((resolve, reject) => { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { if (!name || typeof name !== 'string' || name.trim() === '') { - console.error('ERROR: Invalid font name: %s', name); + console.error('ERROR: Invalid font name: %s', String(name)); return reject('Invalid font name'); } if (!/^\d+-\d+$/.test(range)) { From 6ab84eb125ec9cbe64815cb92673773d8c3aa795 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 14:31:39 -0500 Subject: [PATCH 048/104] font fixes --- src/utils.js | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/utils.js b/src/utils.js index d9133a805..d0ebd8597 100644 --- a/src/utils.js +++ b/src/utils.js @@ -196,16 +196,19 @@ export function fixTileJSONCenter(tileJSON) { */ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return new Promise((resolve, reject) => { + const fontMatch = name?.match(/^[\w\s-]+$/); + if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { + console.error('ERROR: Invalid font name: %s', 'invalid'); + return reject('Invalid font name'); + } + const sanitizedName = fontMatch[0]; + const filename = path.join(fontPath, sanitizedName, `${range}.pbf`); + + if (!/^\d+-\d+$/.test(range)) { + console.error('ERROR: Invalid range: %s', range); + return reject('Invalid range'); + } if (!allowedFonts || (allowedFonts[name] && fallbacks)) { - if (!name || typeof name !== 'string' || name.trim() === '') { - console.error('ERROR: Invalid font name: %s', String(name)); - return reject('Invalid font name'); - } - if (!/^\d+-\d+$/.test(range)) { - console.error('ERROR: Invalid range: %s', range); - return reject('Invalid range'); - } - const filename = path.join(fontPath, name, `${range}.pbf`); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } @@ -213,11 +216,15 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { - console.error('ERROR: Font not found: %s, Error: %s', filename, err); + console.error( + 'ERROR: Font not found: %s, Error: %s', + filename, + String(err), + ); if (fallbacks && Object.keys(fallbacks).length) { let fallbackName; - let fontStyle = name.split(' ').pop(); + let fontStyle = sanitizedName.split(' ').pop(); if (['Regular', 'Bold', 'Italic'].indexOf(fontStyle) < 0) { fontStyle = 'Regular'; } @@ -228,10 +235,10 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { fallbackName = Object.keys(fallbacks)[0]; } } - console.error( - `ERROR: Trying to use %s as a fallback`, + `ERROR: Trying to use %s as a fallback for: %s`, fallbackName, + sanitizedName, ); delete fallbacks[fallbackName]; getFontPbf(null, fontPath, fallbackName, range, fallbacks).then( @@ -239,14 +246,14 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { reject, ); } else { - reject(`Font load error: ${name}`); + reject('Font load error'); } } else { resolve(data); } }); } else { - reject(`Font not allowed: ${name}`); + reject('Font not allowed'); } }); } From a8053028a8affa3287c984df1811e210eaf4c1b4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 14:55:24 -0500 Subject: [PATCH 049/104] fix tile size issue --- src/serve_rendered.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index e7055aedd..66e25e98e 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -674,12 +674,12 @@ async function handleTileRequest( const y = parseFloat(yParam) | 0; const scale = parseScale(scaleParam, maxScaleFactor); - let parsedTileSize = defailtTileSize; + let parsedTileSize = parseInt(defailtTileSize, 10); if (tileSize) { const allowedTileSizes = allowedOptions(['256', '512'], { defaultValue: null, }); - parsedTileSize = allowedTileSizes(tileSize); + parsedTileSize = parseInt(allowedTileSizes(tileSize), 10); if (parsedTileSize == null) { return res.status(400).send('Invalid Tile Size'); From 5f85802aedb9279b5f0dc0156bfe5f4dc346a07a Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 23:09:05 -0500 Subject: [PATCH 050/104] try to improve scale + codeql --- src/serve_style.js | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 80f42e6f3..15156e521 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -13,17 +13,25 @@ const httpTester = /^https?:\/\//i; const allowedSpriteFormats = allowedOptions(['png', 'json']); /** - * Checks and formats sprite scale - * @param {string} scale string containing the scale - * @returns {string} formated string for the scale or empty string if scale is invalid + * Checks if a string is a valid sprite scale and returns it if it is within the allowed range, and null if it does not conform. + * @param {string} scale - The scale string to validate (e.g., '2x', '3x'). + * @param {number} [maxScale] - The maximum scale value. If no value is passed in, it defaults to a value of 3. + * @returns {string|null} - The valid scale string or null if invalid. */ -function allowedSpriteScales(scale) { - if (!scale) return ''; - const match = scale.match(/(\d+)x/); - const parsedScale = match ? parseInt(match[1], 10) : 1; - return '@' + Math.min(parsedScale, 3) + 'x'; +function allowedSpriteScales(scale, maxScale = 3) { + if (!scale) { + return ''; + } + const match = scale?.match(/^([2-9]\d*)x$/); + if (!match) { + return null; + } + const parsedScale = parseInt(match[1], 10); + if (parsedScale <= maxScale) { + return `@${parsedScale}x`; + } + return null; } - export const serve_style = { /** * Initializes the serve_style module. @@ -92,26 +100,27 @@ export const serve_style = { */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { const { spriteID = 'default', id, format, scale } = req.params; - const spriteScale = allowedSpriteScales(scale); - if (verbose) { console.log( - `Handling sprite request for: /${id}/sprite/${spriteID}${scale}.${format}`, + `Handling sprite request for: /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, ); } + const item = repo[id]; - if (!item || !allowedSpriteFormats(format)) { + const spriteScale = allowedSpriteScales(scale); + if (!item || !allowedSpriteFormats(format) || spriteScale === null) { if (verbose) console.error( - `Sprite item or format not found for: /${id}/sprite/${spriteID}${scale}.${format}`, + `Sprite item, format, or scale not found for: /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, ); return res.sendStatus(404); } + const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); if (!sprite) { if (verbose) console.error( - `Sprite not found for: /${id}/sprite/${spriteID}${scale}.${format}`, + `Sprite not found for: /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, ); return res.status(400).send('Bad Sprite ID or Scale'); } @@ -134,7 +143,7 @@ export const serve_style = { } if (verbose) console.log( - `Responding with sprite data for /${id}/sprite/${spriteID}${scale}.${format}`, + `Responding with sprite data for /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, ); return res.send(data); }); From b825c9a21bb2421d1039d19ec2209e23fe4ad263 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 23:28:28 -0500 Subject: [PATCH 051/104] codeql for sprite logging --- src/serve_style.js | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 15156e521..866e1f556 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -102,37 +102,52 @@ export const serve_style = { const { spriteID = 'default', id, format, scale } = req.params; if (verbose) { console.log( - `Handling sprite request for: /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, + `Handling sprite request for: /styles/%s/sprite/%s%s%s`, + id, + spriteID, + scale ? scale : '', + format ? '.' + format : '', ); } - const item = repo[id]; - const spriteScale = allowedSpriteScales(scale); - if (!item || !allowedSpriteFormats(format) || spriteScale === null) { + const validatedFormat = allowedSpriteFormats(format); + if (!item || !validatedFormat) { if (verbose) console.error( - `Sprite item, format, or scale not found for: /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, + `Sprite item, format, or scale not found for: /styles/%s/sprite/%s%s%s`, + id, + spriteID, + scale ? scale : '', + format ? '.' + format : '', ); return res.sendStatus(404); } - + const spriteScale = allowedSpriteScales(scale); const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); - if (!sprite) { + if (!sprite || spriteScale === null) { if (verbose) console.error( - `Sprite not found for: /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, + `Sprite not found for: /styles/%s/sprite/%s%s%s`, + id, + spriteID, + scale ? scale : '', + format ? '.' + format : '', ); return res.status(400).send('Bad Sprite ID or Scale'); } - const filename = `${sprite.path}${spriteScale}.${format}`; - if (verbose) console.log(`Loading sprite from: ${filename}`); + const filename = `${sprite.path}${spriteScale}.${validatedFormat}`; + if (verbose) console.log(`Loading sprite from: %s`, filename); // eslint-disable-next-line security/detect-non-literal-fs-filename fs.readFile(filename, (err, data) => { if (err) { if (verbose) - console.error('Sprite load error: %s, Error: %s', filename, err); + console.error( + 'Sprite load error: %s, Error: %s', + filename, + String(err), + ); return res.sendStatus(404); } @@ -143,7 +158,11 @@ export const serve_style = { } if (verbose) console.log( - `Responding with sprite data for /styles/${id}/sprite/${spriteID}${scale ? scale : ''}${format ? '.' + format : ''}`, + `Responding with sprite data for /styles/%s/sprite/%s%s%s`, + id, + spriteID, + scale ? scale : '', + format ? '.' + format : '', ); return res.send(data); }); From e1cae331b9bd031f0be944938e83b9ade7ff914e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 3 Jan 2025 23:40:30 -0500 Subject: [PATCH 052/104] codeql serve fonts --- src/serve_font.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 020d6d585..ec0b424fb 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -32,10 +32,18 @@ export async function serve_font(options, allowedFonts, programOpts) { app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { if (verbose) { console.log( - `Handling font request for: /fonts/${req.params.fontstack}/${req.params.range}.pbf`, + `Handling font request for: /fonts/%s/%s.pbf`, + req.params.fontstack, + req.params.range, ); } - const fontstack = decodeURI(req.params.fontstack); + let fontstack = req.params.fontstack; + const fontStackMatch = fontstack?.match(/^[\w\s-]+$/); + if (!fontStackMatch) { + return res.status(400).send('Invalid font stack format'); + } + fontstack = decodeURI(fontStackMatch[0]); + const range = req.params.range; try { @@ -51,7 +59,12 @@ export async function serve_font(options, allowedFonts, programOpts) { res.header('Last-Modified', lastModified); return res.send(concatenated); } catch (err) { - console.error(`Error serving font: ${fontstack}/${range}.pbf`, err); + console.error( + `Error serving font: %s/%s.pbf, Error: %s`, + fontstack, + range, + String(err), + ); return res .status(400) .header('Content-Type', 'text/plain') From e6750dca5a2d9b5b8e59e7fbf3d9b9d4a9988f19 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 00:13:53 -0500 Subject: [PATCH 053/104] codeql fixes --- src/serve_rendered.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 66e25e98e..fb2a3198b 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -942,7 +942,6 @@ export const serve_rendered = { * @param {string} req.params.p2 - The z, static, or raw parameter. * @param {string} req.params.p3 - The x or staticType parameter. * @param {string} req.params.p4 - The y or width parameter. - * @param {string} req.params.p5 - The height parameter. * @param {string} req.params.scale - The scale parameter. * @param {string} req.params.format - The format of the image. * @returns {Promise} @@ -951,17 +950,22 @@ export const serve_rendered = { `/:id{/:p1}/:p2/:p3/:p4{@:scale}{.:format}`, async (req, res, next) => { try { - const { p1, p2, id, p3, p4, p5, scale, format } = req.params; + const { p1, p2, id, p3, p4, scale, format } = req.params; const requestType = (!p1 && p2 === 'static') || (p1 === 'static' && p2 === 'raw') ? 'static' : 'tile'; - if (verbose) { console.log( - `Handling rendered ${requestType} request for: /styles/${id}${p1 ? '/' + p1 : ''}/${p2}/${p3}/${p4}${p5 ? 'x' + p5 : ''}${ - scale ? '@' + scale : '' - }.${format}`, + `Handling rendered %s request for: /styles/%s%s/%s/%s/%s%s.%s`, + requestType, + id, + p1 ? '/' + p1 : '', + p2, + p3, + p4, + scale ? '@' + scale : '', + format, ); } From 8420ef16a74003f0ecb711bc481d58f9de698b74 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 00:14:10 -0500 Subject: [PATCH 054/104] fix failing test with multiple fonts --- src/serve_font.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index ec0b424fb..546e486a0 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -38,12 +38,18 @@ export async function serve_font(options, allowedFonts, programOpts) { ); } let fontstack = req.params.fontstack; - const fontStackMatch = fontstack?.match(/^[\w\s-]+$/); - if (!fontStackMatch) { + const fontStackParts = fontstack.split(','); + const sanitizedFontStack = fontStackParts + .map((font) => { + const fontMatch = font?.match(/^[\w\s-]+$/); + return fontMatch?.[0]; + }) + .filter(Boolean) + .join(','); + if (sanitizedFontStack.length == 0) { return res.status(400).send('Invalid font stack format'); } - fontstack = decodeURI(fontStackMatch[0]); - + fontstack = decodeURI(sanitizedFontStack); const range = req.params.range; try { From faf01d784fb6d015f13db9ec2d86f1d1221d74aa Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 00:36:11 -0500 Subject: [PATCH 055/104] Update serve_font.js --- src/serve_font.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 546e486a0..65587dc6c 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -30,14 +30,8 @@ export async function serve_font(options, allowedFonts, programOpts) { * @returns {Promise} */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { - if (verbose) { - console.log( - `Handling font request for: /fonts/%s/%s.pbf`, - req.params.fontstack, - req.params.range, - ); - } let fontstack = req.params.fontstack; + let range = req.params.range; const fontStackParts = fontstack.split(','); const sanitizedFontStack = fontStackParts .map((font) => { @@ -49,8 +43,15 @@ export async function serve_font(options, allowedFonts, programOpts) { if (sanitizedFontStack.length == 0) { return res.status(400).send('Invalid font stack format'); } + + if (verbose) { + console.log( + `Handling font request for: /fonts/%s/%s.pbf`, + sanitizedFontStack, + range, + ); + } fontstack = decodeURI(sanitizedFontStack); - const range = req.params.range; try { const concatenated = await getFontsPbf( From 3517060488de595ef8a548299dd7d64eac56c725 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 00:42:11 -0500 Subject: [PATCH 056/104] codeql --- src/serve_font.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 65587dc6c..680d09480 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -69,7 +69,7 @@ export async function serve_font(options, allowedFonts, programOpts) { console.error( `Error serving font: %s/%s.pbf, Error: %s`, fontstack, - range, + String(range), String(err), ); return res @@ -77,7 +77,7 @@ export async function serve_font(options, allowedFonts, programOpts) { .header('Content-Type', 'text/plain') .send('Error serving font'); } - }); + }); /** * Handles requests for a list of all available fonts. From d4aaa6268e7ef2f2b30208fc6212a7e878787a6b Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 00:43:49 -0500 Subject: [PATCH 057/104] codeql --- src/serve_font.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serve_font.js b/src/serve_font.js index 680d09480..d9741b361 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -48,7 +48,7 @@ export async function serve_font(options, allowedFonts, programOpts) { console.log( `Handling font request for: /fonts/%s/%s.pbf`, sanitizedFontStack, - range, + String(range), ); } fontstack = decodeURI(sanitizedFontStack); From afa59521fa2938b60ee80671292be15c2f0e5b61 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 01:26:42 -0500 Subject: [PATCH 058/104] codeql --- src/serve_style.js | 11 +++++++---- src/utils.js | 37 ++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 866e1f556..8e6b74f49 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -122,8 +122,9 @@ export const serve_style = { ); return res.sendStatus(404); } - const spriteScale = allowedSpriteScales(scale); + const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); + const spriteScale = allowedSpriteScales(scale); if (!sprite || spriteScale === null) { if (verbose) console.error( @@ -136,7 +137,9 @@ export const serve_style = { return res.status(400).send('Bad Sprite ID or Scale'); } - const filename = `${sprite.path}${spriteScale}.${validatedFormat}`; + const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); + + const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; if (verbose) console.log(`Loading sprite from: %s`, filename); // eslint-disable-next-line security/detect-non-literal-fs-filename @@ -151,9 +154,9 @@ export const serve_style = { return res.sendStatus(404); } - if (format === 'json') { + if (validatedFormat === 'json') { res.header('Content-type', 'application/json'); - } else if (format === 'png') { + } else if (validatedFormat === 'png') { res.header('Content-type', 'image/png'); } if (verbose) diff --git a/src/utils.js b/src/utils.js index d0ebd8597..7f85bca74 100644 --- a/src/utils.js +++ b/src/utils.js @@ -196,19 +196,29 @@ export function fixTileJSONCenter(tileJSON) { */ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return new Promise((resolve, reject) => { - const fontMatch = name?.match(/^[\w\s-]+$/); - if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { - console.error('ERROR: Invalid font name: %s', 'invalid'); - return reject('Invalid font name'); - } - const sanitizedName = fontMatch[0]; - const filename = path.join(fontPath, sanitizedName, `${range}.pbf`); - - if (!/^\d+-\d+$/.test(range)) { - console.error('ERROR: Invalid range: %s', range); - return reject('Invalid range'); - } if (!allowedFonts || (allowedFonts[name] && fallbacks)) { + const fontMatch = name?.match(/^[\w\s-]+$/); + if ( + !name || + typeof name !== 'string' || + name.trim() === '' || + !fontMatch + ) { + console.error('ERROR: Invalid font name: %s', 'invalid'); + return reject('Invalid font name'); + } + const sanitizedName = fontMatch[0]; + console.error('ERROR: Invalid font name: %s', sanitizedName); + if (!/^\d+-\d+$/.test(range)) { + console.error('ERROR: Invalid range: %s', range); + return reject('Invalid range'); + } + const sanitizedFontPath = fontPath.replace(/^(\.\.\/)+/, ''); + const filename = path.join( + sanitizedFontPath, + sanitizedName, + `${range}.pbf`, + ); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } @@ -224,7 +234,7 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (fallbacks && Object.keys(fallbacks).length) { let fallbackName; - let fontStyle = sanitizedName.split(' ').pop(); + let fontStyle = name.split(' ').pop(); if (['Regular', 'Bold', 'Italic'].indexOf(fontStyle) < 0) { fontStyle = 'Regular'; } @@ -235,6 +245,7 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { fallbackName = Object.keys(fallbacks)[0]; } } + console.error( `ERROR: Trying to use %s as a fallback for: %s`, fallbackName, From 2be472b9414966c107b2e6c199c67a2ffd660786 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 02:10:30 -0500 Subject: [PATCH 059/104] Update utils.js --- src/utils.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/utils.js b/src/utils.js index 7f85bca74..0566dcbff 100644 --- a/src/utils.js +++ b/src/utils.js @@ -198,27 +198,22 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return new Promise((resolve, reject) => { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { const fontMatch = name?.match(/^[\w\s-]+$/); + const sanitizedName = fontMatch?.[0] || 'invalid'; if ( !name || typeof name !== 'string' || name.trim() === '' || !fontMatch ) { - console.error('ERROR: Invalid font name: %s', 'invalid'); + console.error('ERROR: Invalid font name: %s', sanitizedName); return reject('Invalid font name'); } - const sanitizedName = fontMatch[0]; - console.error('ERROR: Invalid font name: %s', sanitizedName); + if (!/^\d+-\d+$/.test(range)) { console.error('ERROR: Invalid range: %s', range); return reject('Invalid range'); } - const sanitizedFontPath = fontPath.replace(/^(\.\.\/)+/, ''); - const filename = path.join( - sanitizedFontPath, - sanitizedName, - `${range}.pbf`, - ); + const filename = path.join(fontPath, sanitizedName, `${range}.pbf`); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } @@ -245,7 +240,6 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { fallbackName = Object.keys(fallbacks)[0]; } } - console.error( `ERROR: Trying to use %s as a fallback for: %s`, fallbackName, @@ -268,7 +262,6 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } }); } - /** * Combines multiple font pbf buffers into one. * @param {object} allowedFonts - An object of allowed fonts. From f6b24a591623088e5624c23b8a17e82dba532a53 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 02:19:36 -0500 Subject: [PATCH 060/104] codeql --- src/serve_font.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index d9741b361..dea3f07e9 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -30,8 +30,14 @@ export async function serve_font(options, allowedFonts, programOpts) { * @returns {Promise} */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + if (verbose) { + console.log( + `Handling font request for: /fonts/%s/%s.pbf`, + req.params.fontstack, + req.params.range, + ); + } let fontstack = req.params.fontstack; - let range = req.params.range; const fontStackParts = fontstack.split(','); const sanitizedFontStack = fontStackParts .map((font) => { @@ -43,15 +49,10 @@ export async function serve_font(options, allowedFonts, programOpts) { if (sanitizedFontStack.length == 0) { return res.status(400).send('Invalid font stack format'); } - - if (verbose) { - console.log( - `Handling font request for: /fonts/%s/%s.pbf`, - sanitizedFontStack, - String(range), - ); - } fontstack = decodeURI(sanitizedFontStack); + let range = req.params.range; + const rangeMatch = range?.match(/^[\d-]+$/); + const sanitizedRange = rangeMatch?.[0] || 'invalid'; try { const concatenated = await getFontsPbf( @@ -69,7 +70,7 @@ export async function serve_font(options, allowedFonts, programOpts) { console.error( `Error serving font: %s/%s.pbf, Error: %s`, fontstack, - String(range), + sanitizedRange, String(err), ); return res @@ -77,7 +78,7 @@ export async function serve_font(options, allowedFonts, programOpts) { .header('Content-Type', 'text/plain') .send('Error serving font'); } - }); + }); /** * Handles requests for a list of all available fonts. From f187d32a4a600d85128299a178b05d99d019b1ba Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 02:27:30 -0500 Subject: [PATCH 061/104] codeql --- src/serve_font.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index dea3f07e9..3b8b68f8b 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -30,13 +30,6 @@ export async function serve_font(options, allowedFonts, programOpts) { * @returns {Promise} */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { - if (verbose) { - console.log( - `Handling font request for: /fonts/%s/%s.pbf`, - req.params.fontstack, - req.params.range, - ); - } let fontstack = req.params.fontstack; const fontStackParts = fontstack.split(','); const sanitizedFontStack = fontStackParts @@ -53,6 +46,13 @@ export async function serve_font(options, allowedFonts, programOpts) { let range = req.params.range; const rangeMatch = range?.match(/^[\d-]+$/); const sanitizedRange = rangeMatch?.[0] || 'invalid'; + if (verbose) { + console.log( + `Handling font request for: /fonts/%s/%s.pbf`, + sanitizedFontStack, + sanitizedRange, + ); + } try { const concatenated = await getFontsPbf( @@ -70,7 +70,7 @@ export async function serve_font(options, allowedFonts, programOpts) { console.error( `Error serving font: %s/%s.pbf, Error: %s`, fontstack, - sanitizedRange, + String(range), String(err), ); return res From 4abc6164b7f352f39c0cfe6ff7302e122ddc8ff4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 02:35:51 -0500 Subject: [PATCH 062/104] codeql --- src/serve_style.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serve_style.js b/src/serve_style.js index 8e6b74f49..4adf6587b 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -54,7 +54,7 @@ export const serve_style = { app.get('/:id/style.json', (req, res, next) => { const { id } = req.params; if (verbose) { - console.log(`Handling style request for: /styles/${id}/style.json`); + console.log('Handling style request for: /styles/%s/style.json', id); } try { const item = repo[id]; From bc85d7a1e57611fce35f3bb6e94522d613f74870 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 02:49:01 -0500 Subject: [PATCH 063/104] codeql --- src/serve_style.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 4adf6587b..4f0588961 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -100,13 +100,16 @@ export const serve_style = { */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { const { spriteID = 'default', id, format, scale } = req.params; + const sanitizedScale = scale ? String(scale) : ''; + const sanitizedSpriteID = String(spriteID); + const sanitizedFormat = format ? '.' + String(format) : ''; if (verbose) { console.log( `Handling sprite request for: /styles/%s/sprite/%s%s%s`, id, - spriteID, - scale ? scale : '', - format ? '.' + format : '', + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, ); } const item = repo[id]; @@ -116,9 +119,9 @@ export const serve_style = { console.error( `Sprite item, format, or scale not found for: /styles/%s/sprite/%s%s%s`, id, - spriteID, - scale ? scale : '', - format ? '.' + format : '', + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, ); return res.sendStatus(404); } @@ -130,15 +133,14 @@ export const serve_style = { console.error( `Sprite not found for: /styles/%s/sprite/%s%s%s`, id, - spriteID, - scale ? scale : '', - format ? '.' + format : '', + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, ); return res.status(400).send('Bad Sprite ID or Scale'); } const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); - const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; if (verbose) console.log(`Loading sprite from: %s`, filename); @@ -163,9 +165,9 @@ export const serve_style = { console.log( `Responding with sprite data for /styles/%s/sprite/%s%s%s`, id, - spriteID, - scale ? scale : '', - format ? '.' + format : '', + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, ); return res.send(data); }); From 4fe9bda0705f5019124742ef47def46394f1d8b4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 03:00:50 -0500 Subject: [PATCH 064/104] codeql sanitize --- src/serve_style.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 4f0588961..5cad141b1 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -100,13 +100,14 @@ export const serve_style = { */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { const { spriteID = 'default', id, format, scale } = req.params; + const sanitizedId = String(id); const sanitizedScale = scale ? String(scale) : ''; const sanitizedSpriteID = String(spriteID); const sanitizedFormat = format ? '.' + String(format) : ''; if (verbose) { console.log( `Handling sprite request for: /styles/%s/sprite/%s%s%s`, - id, + sanitizedId, sanitizedSpriteID, sanitizedScale, sanitizedFormat, @@ -118,7 +119,7 @@ export const serve_style = { if (verbose) console.error( `Sprite item, format, or scale not found for: /styles/%s/sprite/%s%s%s`, - id, + sanitizedId, sanitizedSpriteID, sanitizedScale, sanitizedFormat, @@ -132,7 +133,7 @@ export const serve_style = { if (verbose) console.error( `Sprite not found for: /styles/%s/sprite/%s%s%s`, - id, + sanitizedId, sanitizedSpriteID, sanitizedScale, sanitizedFormat, @@ -155,7 +156,6 @@ export const serve_style = { ); return res.sendStatus(404); } - if (validatedFormat === 'json') { res.header('Content-type', 'application/json'); } else if (validatedFormat === 'png') { @@ -164,7 +164,7 @@ export const serve_style = { if (verbose) console.log( `Responding with sprite data for /styles/%s/sprite/%s%s%s`, - id, + sanitizedId, sanitizedSpriteID, sanitizedScale, sanitizedFormat, From 7a81e47c09072d1fe5e4338d07595477053f7b9b Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 03:11:05 -0500 Subject: [PATCH 065/104] Update serve_font.js --- src/serve_font.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serve_font.js b/src/serve_font.js index 3b8b68f8b..7c383008d 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -70,7 +70,7 @@ export async function serve_font(options, allowedFonts, programOpts) { console.error( `Error serving font: %s/%s.pbf, Error: %s`, fontstack, - String(range), + sanitizedRange, String(err), ); return res From 0dcf2964252f65641a624211431fef85148855bb Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 11:23:35 -0500 Subject: [PATCH 066/104] Update serve_font.js --- src/serve_font.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serve_font.js b/src/serve_font.js index 7c383008d..440355358 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -35,7 +35,7 @@ export async function serve_font(options, allowedFonts, programOpts) { const sanitizedFontStack = fontStackParts .map((font) => { const fontMatch = font?.match(/^[\w\s-]+$/); - return fontMatch?.[0]; + return fontMatch?.[0] || 'invalid'; }) .filter(Boolean) .join(','); From 25052493cb1f0fb1dc9b1a668a65551771f05964 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 11:48:55 -0500 Subject: [PATCH 067/104] remove useless assignment --- src/serve_rendered.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index fb2a3198b..5155787be 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -1153,7 +1153,6 @@ export const serve_rendered = { if (format === 'pbf') { if (isGzipped) { response.data = await gunzipP(response.data); - isGzipped = false; } if (options.dataDecoratorFunc) { response.data = options.dataDecoratorFunc( From 4e2e46a98d854d9e3d13520adef5997918f67a83 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 11:54:54 -0500 Subject: [PATCH 068/104] move isGzipped --- src/serve_rendered.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 5155787be..e3047ee56 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -1142,15 +1142,16 @@ export const serve_rendered = { const response = {}; response.data = fetchTile.data; let headers = fetchTile.headers; - let isGzipped = - response.data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === - 0; if (headers['Last-Modified']) { response.modified = new Date(headers['Last-Modified']); } if (format === 'pbf') { + let isGzipped = + response.data + .slice(0, 2) + .indexOf(Buffer.from([0x1f, 0x8b])) === 0; if (isGzipped) { response.data = await gunzipP(response.data); } From 7e67f40fbf4e5e47b5981be3b2d49fca2ce69955 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 12:50:37 -0500 Subject: [PATCH 069/104] add if-modified-since and cache-control --- src/serve_style.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 5cad141b1..bfc1a4588 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -118,7 +118,7 @@ export const serve_style = { if (!item || !validatedFormat) { if (verbose) console.error( - `Sprite item, format, or scale not found for: /styles/%s/sprite/%s%s%s`, + `Sprite item or format not found for: /styles/%s/sprite/%s%s%s`, sanitizedId, sanitizedSpriteID, sanitizedScale, @@ -126,13 +126,12 @@ export const serve_style = { ); return res.sendStatus(404); } - const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); const spriteScale = allowedSpriteScales(scale); if (!sprite || spriteScale === null) { if (verbose) console.error( - `Sprite not found for: /styles/%s/sprite/%s%s%s`, + `Bad Sprite ID or Scale for: /styles/%s/sprite/%s%s%s`, sanitizedId, sanitizedSpriteID, sanitizedScale, @@ -141,6 +140,14 @@ export const serve_style = { return res.status(400).send('Bad Sprite ID or Scale'); } + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if (new Date(item.lastModified) <= new Date(modifiedSince)) { + return res.sendStatus(304); + } + } + const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; if (verbose) console.log(`Loading sprite from: %s`, filename); @@ -156,6 +163,7 @@ export const serve_style = { ); return res.sendStatus(404); } + if (validatedFormat === 'json') { res.header('Content-type', 'application/json'); } else if (validatedFormat === 'png') { @@ -169,6 +177,7 @@ export const serve_style = { sanitizedScale, sanitizedFormat, ); + res.set({ 'Last-Modified': item.lastModified }); return res.send(data); }); }); From 23f50d01a69338daded7babc0ad2fdd3257acc45 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 14:43:06 -0500 Subject: [PATCH 070/104] use consistent cache control --- src/serve_font.js | 11 +++++++++++ src/serve_rendered.js | 4 +++- src/serve_style.js | 5 ++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 440355358..d8ac041e6 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -42,6 +42,7 @@ export async function serve_font(options, allowedFonts, programOpts) { if (sanitizedFontStack.length == 0) { return res.status(400).send('Invalid font stack format'); } + fontstack = decodeURI(sanitizedFontStack); let range = req.params.range; const rangeMatch = range?.match(/^[\d-]+$/); @@ -54,6 +55,16 @@ export async function serve_font(options, allowedFonts, programOpts) { ); } + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + const lastDate = new Date(lastModified).getTime(); + const modDate = new Date(modifiedSince).getTime(); + if (lastDate === modDate) { + return res.sendStatus(304); + } + } + try { const concatenated = await getFontsPbf( options.serveAllFonts ? null : allowedFonts, diff --git a/src/serve_rendered.js b/src/serve_rendered.js index e3047ee56..ef5cf306e 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -665,7 +665,9 @@ async function handleTileRequest( const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(item.lastModified) <= new Date(modifiedSince)) { + const lastDate = new Date(item.lastModified).getTime(); + const modDate = new Date(modifiedSince).getTime(); + if (lastDate === modDate) { return res.sendStatus(304); } } diff --git a/src/serve_style.js b/src/serve_style.js index bfc1a4588..b6ff0e236 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -143,7 +143,9 @@ export const serve_style = { const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(item.lastModified) <= new Date(modifiedSince)) { + const lastDate = new Date(item.lastModified).getTime(); + const modDate = new Date(modifiedSince).getTime(); + if (lastDate === modDate) { return res.sendStatus(304); } } @@ -327,6 +329,7 @@ export const serve_style = { spritePaths, publicUrl, name: styleJSON.name, + lastModified: new Date().toUTCString(), }; return true; From f888286f00101c44ec8ec89d520c530a78a3b306 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 14:48:17 -0500 Subject: [PATCH 071/104] reformat --- src/serve_font.js | 6 +++--- src/serve_rendered.js | 7 ++++--- src/serve_style.js | 7 ++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index d8ac041e6..f80c4209c 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -58,9 +58,9 @@ export async function serve_font(options, allowedFonts, programOpts) { const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - const lastDate = new Date(lastModified).getTime(); - const modDate = new Date(modifiedSince).getTime(); - if (lastDate === modDate) { + if ( + new Date(lastModified).getTime() === new Date(modifiedSince).getTime() + ) { return res.sendStatus(304); } } diff --git a/src/serve_rendered.js b/src/serve_rendered.js index ef5cf306e..f54cc4977 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -665,9 +665,10 @@ async function handleTileRequest( const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - const lastDate = new Date(item.lastModified).getTime(); - const modDate = new Date(modifiedSince).getTime(); - if (lastDate === modDate) { + if ( + new Date(item.lastModified).getTime() === + new Date(modifiedSince).getTime() + ) { return res.sendStatus(304); } } diff --git a/src/serve_style.js b/src/serve_style.js index b6ff0e236..68c631928 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -143,9 +143,10 @@ export const serve_style = { const modifiedSince = req.get('if-modified-since'); const cc = req.get('cache-control'); if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - const lastDate = new Date(item.lastModified).getTime(); - const modDate = new Date(modifiedSince).getTime(); - if (lastDate === modDate) { + if ( + new Date(item.lastModified).getTime() === + new Date(modifiedSince).getTime() + ) { return res.sendStatus(304); } } From c2f95ab2d733b46c32acb5de14da63341e208462 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 22:47:32 -0500 Subject: [PATCH 072/104] codeql --- src/serve_style.js | 5 ++++- src/utils.js | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 68c631928..63d7df4a3 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -54,7 +54,10 @@ export const serve_style = { app.get('/:id/style.json', (req, res, next) => { const { id } = req.params; if (verbose) { - console.log('Handling style request for: /styles/%s/style.json', id); + console.log( + 'Handling style request for: /styles/%s/style.json', + String(id), + ); } try { const item = repo[id]; diff --git a/src/utils.js b/src/utils.js index 0566dcbff..354b35c82 100644 --- a/src/utils.js +++ b/src/utils.js @@ -209,11 +209,17 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { return reject('Invalid font name'); } + const rangeMatch = range?.match(/^[\d-]+$/); + const sanitizedRange = rangeMatch?.[0] || 'invalid'; if (!/^\d+-\d+$/.test(range)) { - console.error('ERROR: Invalid range: %s', range); + console.error('ERROR: Invalid range: %s', sanitizedRange); return reject('Invalid range'); } - const filename = path.join(fontPath, sanitizedName, `${range}.pbf`); + const filename = path.join( + fontPath, + sanitizedName, + `${sanitizedRange}.pbf`, + ); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } From 3def0d2c4561c3e85b3aa885243c0d2b6df22e31 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 22:58:11 -0500 Subject: [PATCH 073/104] codeql --- src/serve_font.js | 8 ++++---- src/serve_style.js | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index f80c4209c..09f34b1a9 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -50,8 +50,8 @@ export async function serve_font(options, allowedFonts, programOpts) { if (verbose) { console.log( `Handling font request for: /fonts/%s/%s.pbf`, - sanitizedFontStack, - sanitizedRange, + sanitizedFontStack.replace(/\n|\r/g, ''), + sanitizedRange.replace(/\n|\r/g, ''), ); } @@ -80,8 +80,8 @@ export async function serve_font(options, allowedFonts, programOpts) { } catch (err) { console.error( `Error serving font: %s/%s.pbf, Error: %s`, - fontstack, - sanitizedRange, + sanitizedFontStack.replace(/\n|\r/g, ''), + sanitizedRange.replace(/\n|\r/g, ''), String(err), ); return res diff --git a/src/serve_style.js b/src/serve_style.js index 63d7df4a3..36975ec5d 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -56,7 +56,7 @@ export const serve_style = { if (verbose) { console.log( 'Handling style request for: /styles/%s/style.json', - String(id), + String(id).replace(/\n|\r/g, ''), ); } try { @@ -103,10 +103,12 @@ export const serve_style = { */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { const { spriteID = 'default', id, format, scale } = req.params; - const sanitizedId = String(id); - const sanitizedScale = scale ? String(scale) : ''; - const sanitizedSpriteID = String(spriteID); - const sanitizedFormat = format ? '.' + String(format) : ''; + const sanitizedId = String(id).replace(/\n|\r/g, ''); + const sanitizedScale = scale ? String(scale).replace(/\n|\r/g, '') : ''; + const sanitizedSpriteID = String(spriteID).replace(/\n|\r/g, ''); + const sanitizedFormat = format + ? '.' + String(format).replace(/\n|\r/g, '') + : ''; if (verbose) { console.log( `Handling sprite request for: /styles/%s/sprite/%s%s%s`, From 1c5ee75c34724d916b1a2d4ad39da1cf98b23e00 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 23:02:53 -0500 Subject: [PATCH 074/104] codeql --- src/utils.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/utils.js b/src/utils.js index 354b35c82..1a164e755 100644 --- a/src/utils.js +++ b/src/utils.js @@ -205,14 +205,20 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { name.trim() === '' || !fontMatch ) { - console.error('ERROR: Invalid font name: %s', sanitizedName); + console.error( + 'ERROR: Invalid font name: %s', + sanitizedName.replace(/\n|\r/g, ''), + ); return reject('Invalid font name'); } const rangeMatch = range?.match(/^[\d-]+$/); const sanitizedRange = rangeMatch?.[0] || 'invalid'; if (!/^\d+-\d+$/.test(range)) { - console.error('ERROR: Invalid range: %s', sanitizedRange); + console.error( + 'ERROR: Invalid range: %s', + sanitizedRange.replace(/\n|\r/g, ''), + ); return reject('Invalid range'); } const filename = path.join( From 59f4dccb07c7e4f3c9feb2b1e433102f5a617ea4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 23:07:19 -0500 Subject: [PATCH 075/104] codeql --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 1a164e755..a42c2677f 100644 --- a/src/utils.js +++ b/src/utils.js @@ -235,7 +235,7 @@ function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (err) { console.error( 'ERROR: Font not found: %s, Error: %s', - filename, + filename.replace(/\n|\r/g, ''), String(err), ); if (fallbacks && Object.keys(fallbacks).length) { From aa936eaef41c7b0fafc4a98bc7ea1671c04aa906 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 23:32:26 -0500 Subject: [PATCH 076/104] codeql --- src/serve_data.js | 7 ++++++- src/serve_rendered.js | 20 ++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 685f18d82..5bf637d5f 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -47,7 +47,12 @@ export const serve_data = { app.get('/:id/:z/:x/:y.:format', async (req, res) => { if (verbose) { console.log( - `Handling tile request for: /data/${req.params.id}/${req.params.z}/${req.params.x}/${req.params.y}.${req.params.format}`, + `Handling tile request for: /data/%s/%s/%s/%s.%s`, + String(id).replace(/\n|\r/g, ''), + String(z).replace(/\n|\r/g, ''), + String(x).replace(/\n|\r/g, ''), + String(y).replace(/\n|\r/g, ''), + String(format).replace(/\n|\r/g, ''), ); } const item = repo[req.params.id]; diff --git a/src/serve_rendered.js b/src/serve_rendered.js index f54cc4977..850953b3f 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -962,13 +962,13 @@ export const serve_rendered = { console.log( `Handling rendered %s request for: /styles/%s%s/%s/%s/%s%s.%s`, requestType, - id, - p1 ? '/' + p1 : '', - p2, - p3, - p4, - scale ? '@' + scale : '', - format, + String(id).replace(/\n|\r/g, ''), + p1 ? '/' + String(p1).replace(/\n|\r/g, '') : '', + String(p2).replace(/\n|\r/g, ''), + String(p3).replace(/\n|\r/g, ''), + String(p4).replace(/\n|\r/g, ''), + scale ? '@' + String(scale).replace(/\n|\r/g, '') : '', + String(format).replace(/\n|\r/g, ''), ); } @@ -1019,7 +1019,11 @@ export const serve_rendered = { const tileSize = parseInt(req.params.tileSize, 10) || undefined; if (verbose) { console.log( - `Handling rendered tilejson request for: /styles/${tileSize ? tileSize + '/' : ''}${req.params.id}.json`, + `Handling rendered tilejson request for: /styles/%s%s.json`, + req.params.tileSize + ? String(req.params.tileSize).replace(/\n|\r/g, '') + '/' + : '', + String(req.params.id).replace(/\n|\r/g, ''), ); } const info = clone(item.tileJSON); From 1c3baef5eebb44f6339ae32298d7d850c380b664 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 23:35:57 -0500 Subject: [PATCH 077/104] codeql --- src/serve_data.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index 5bf637d5f..d4b520826 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -48,11 +48,11 @@ export const serve_data = { if (verbose) { console.log( `Handling tile request for: /data/%s/%s/%s/%s.%s`, - String(id).replace(/\n|\r/g, ''), - String(z).replace(/\n|\r/g, ''), - String(x).replace(/\n|\r/g, ''), - String(y).replace(/\n|\r/g, ''), - String(format).replace(/\n|\r/g, ''), + String(req.params.id).replace(/\n|\r/g, ''), + String(req.params.z).replace(/\n|\r/g, ''), + String(req.params.x).replace(/\n|\r/g, ''), + String(req.params.y).replace(/\n|\r/g, ''), + String(req.params.format).replace(/\n|\r/g, ''), ); } const item = repo[req.params.id]; From d85226f7d06e0ba94be43489915a553036194de0 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 23:45:31 -0500 Subject: [PATCH 078/104] codeql --- src/serve_data.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/serve_data.js b/src/serve_data.js index d4b520826..75cc667d6 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -165,7 +165,11 @@ export const serve_data = { try { if (verbose) { console.log( - `Handling elevation request for: /data/${req.params.id}/elevation/${req.params.z}/${req.params.x}/${req.params.y}`, + `Handling elevation request for: /data/%s/elevation/%s/%s/%s`, + String(req.params.id).replace(/\n|\r/g, ''), + String(req.params.z).replace(/\n|\r/g, ''), + String(req.params.x).replace(/\n|\r/g, ''), + String(req.params.y).replace(/\n|\r/g, ''), ); } const item = repo?.[req.params.id]; @@ -313,7 +317,8 @@ export const serve_data = { app.get('/:id.json', (req, res) => { if (verbose) { console.log( - `Handling tilejson request for: /data/${req.params.id}.json`, + `Handling tilejson request for: /data/%s.json`, + String(req.params.id).replace(/\n|\r/g, ''), ); } const item = repo[req.params.id]; From 9a98b38db7b2e62b9a2a95bc90d23dcf8bd1718e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sat, 4 Jan 2025 23:54:49 -0500 Subject: [PATCH 079/104] Update serve_font.js --- src/serve_font.js | 47 ++++++----------------------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 09f34b1a9..821f28a13 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -30,41 +30,15 @@ export async function serve_font(options, allowedFonts, programOpts) { * @returns {Promise} */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { - let fontstack = req.params.fontstack; - const fontStackParts = fontstack.split(','); - const sanitizedFontStack = fontStackParts - .map((font) => { - const fontMatch = font?.match(/^[\w\s-]+$/); - return fontMatch?.[0] || 'invalid'; - }) - .filter(Boolean) - .join(','); - if (sanitizedFontStack.length == 0) { - return res.status(400).send('Invalid font stack format'); - } - - fontstack = decodeURI(sanitizedFontStack); - let range = req.params.range; - const rangeMatch = range?.match(/^[\d-]+$/); - const sanitizedRange = rangeMatch?.[0] || 'invalid'; if (verbose) { console.log( `Handling font request for: /fonts/%s/%s.pbf`, - sanitizedFontStack.replace(/\n|\r/g, ''), - sanitizedRange.replace(/\n|\r/g, ''), + String(req.params.fontstack).replace(/\n|\r/g, ''), + String(req.params.range).replace(/\n|\r/g, ''), ); } - - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if ( - new Date(lastModified).getTime() === new Date(modifiedSince).getTime() - ) { - return res.sendStatus(304); - } - } - + const fontstack = decodeURI(req.params.fontstack); + const range = req.params.range; try { const concatenated = await getFontsPbf( options.serveAllFonts ? null : allowedFonts, @@ -73,21 +47,12 @@ export async function serve_font(options, allowedFonts, programOpts) { range, existingFonts, ); - res.header('Content-type', 'application/x-protobuf'); res.header('Last-Modified', lastModified); return res.send(concatenated); } catch (err) { - console.error( - `Error serving font: %s/%s.pbf, Error: %s`, - sanitizedFontStack.replace(/\n|\r/g, ''), - sanitizedRange.replace(/\n|\r/g, ''), - String(err), - ); - return res - .status(400) - .header('Content-Type', 'text/plain') - .send('Error serving font'); + console.error('Error serving font:', err); + return res.status(400).header('Content-Type', 'text/plain').send(err); } }); From e79e011ae86ac599e31e106a65b89d163a996cca Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 00:03:14 -0500 Subject: [PATCH 080/104] Update serve_font.js --- src/serve_font.js | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index 821f28a13..b24094682 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -30,13 +30,26 @@ export async function serve_font(options, allowedFonts, programOpts) { * @returns {Promise} */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + const sFontStack = String(req.params.fontstack).replace(/\n|\r/g, ''); + const sRange = String(req.params.range).replace(/\n|\r/g, ''); if (verbose) { console.log( `Handling font request for: /fonts/%s/%s.pbf`, - String(req.params.fontstack).replace(/\n|\r/g, ''), - String(req.params.range).replace(/\n|\r/g, ''), + sFontStack, + sRange, ); } + + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if ( + new Date(lastModified).getTime() === new Date(modifiedSince).getTime() + ) { + return res.sendStatus(304); + } + } + const fontstack = decodeURI(req.params.fontstack); const range = req.params.range; try { @@ -51,8 +64,16 @@ export async function serve_font(options, allowedFonts, programOpts) { res.header('Last-Modified', lastModified); return res.send(concatenated); } catch (err) { - console.error('Error serving font:', err); - return res.status(400).header('Content-Type', 'text/plain').send(err); + console.error( + `Error serving font: %s/%s.pbf, Error: %s`, + sFontStack, + sRange, + String(err), + ); + return res + .status(400) + .header('Content-Type', 'text/plain') + .send('Error serving font'); } }); From dce77327167604c7668d4f278e1f7c4e30f3effa Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 00:08:04 -0500 Subject: [PATCH 081/104] Update serve_font.js --- src/serve_font.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/serve_font.js b/src/serve_font.js index b24094682..a42246bed 100644 --- a/src/serve_font.js +++ b/src/serve_font.js @@ -30,8 +30,12 @@ export async function serve_font(options, allowedFonts, programOpts) { * @returns {Promise} */ app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { - const sFontStack = String(req.params.fontstack).replace(/\n|\r/g, ''); const sRange = String(req.params.range).replace(/\n|\r/g, ''); + const sFontStack = String(decodeURI(req.params.fontstack)).replace( + /\n|\r/g, + '', + ); + if (verbose) { console.log( `Handling font request for: /fonts/%s/%s.pbf`, @@ -50,14 +54,12 @@ export async function serve_font(options, allowedFonts, programOpts) { } } - const fontstack = decodeURI(req.params.fontstack); - const range = req.params.range; try { const concatenated = await getFontsPbf( options.serveAllFonts ? null : allowedFonts, fontPath, - fontstack, - range, + sFontStack, + sRange, existingFonts, ); res.header('Content-type', 'application/x-protobuf'); From 0f3629c75287a098e2294d30cd57ed5cdb9fc186 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 00:41:32 -0500 Subject: [PATCH 082/104] Update serve_style.js --- src/serve_style.js | 115 ++++++++++----------------------------------- 1 file changed, 24 insertions(+), 91 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 36975ec5d..5dcdbe513 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -91,103 +91,36 @@ export const serve_style = { }); /** - * Handles GET requests for sprite images and JSON files. - * @param {express.Request} req - Express request object. - * @param {express.Response} res - Express response object. - * @param {express.NextFunction} next - Express next function. - * @param {string} req.params.id - ID of the sprite. - * @param {string} [req.params.spriteID='default'] - ID of the specific sprite image, defaults to 'default'. - * @param {string} [req.params.scale] - Scale of the sprite image, defaults to ''. - * @param {string} req.params.format - Format of the sprite file, 'png' or 'json'. + * Handles requests for a font file. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.fontstack - Name of the font stack. + * @param {string} req.params.range - The range of the font (e.g. 0-255). * @returns {Promise} */ - app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { - const { spriteID = 'default', id, format, scale } = req.params; - const sanitizedId = String(id).replace(/\n|\r/g, ''); - const sanitizedScale = scale ? String(scale).replace(/\n|\r/g, '') : ''; - const sanitizedSpriteID = String(spriteID).replace(/\n|\r/g, ''); - const sanitizedFormat = format - ? '.' + String(format).replace(/\n|\r/g, '') - : ''; + app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { if (verbose) { - console.log( - `Handling sprite request for: /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - } - const item = repo[id]; - const validatedFormat = allowedSpriteFormats(format); - if (!item || !validatedFormat) { - if (verbose) - console.error( - `Sprite item or format not found for: /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - return res.sendStatus(404); - } - const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); - const spriteScale = allowedSpriteScales(scale); - if (!sprite || spriteScale === null) { - if (verbose) - console.error( - `Bad Sprite ID or Scale for: /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - return res.status(400).send('Bad Sprite ID or Scale'); + console.log(req.params); } + const fontstack = decodeURI(req.params.fontstack); + const range = req.params.range; - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if ( - new Date(item.lastModified).getTime() === - new Date(modifiedSince).getTime() - ) { - return res.sendStatus(304); - } - } - - const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); - const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; - if (verbose) console.log(`Loading sprite from: %s`, filename); - - // eslint-disable-next-line security/detect-non-literal-fs-filename - fs.readFile(filename, (err, data) => { - if (err) { - if (verbose) - console.error( - 'Sprite load error: %s, Error: %s', - filename, - String(err), - ); - return res.sendStatus(404); - } + try { + const concatenated = await getFontsPbf( + options.serveAllFonts ? null : allowedFonts, + fontPath, + fontstack, + range, + existingFonts, + ); - if (validatedFormat === 'json') { - res.header('Content-type', 'application/json'); - } else if (validatedFormat === 'png') { - res.header('Content-type', 'image/png'); - } - if (verbose) - console.log( - `Responding with sprite data for /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - res.set({ 'Last-Modified': item.lastModified }); - return res.send(data); - }); + res.header('Content-type', 'application/x-protobuf'); + res.header('Last-Modified', lastModified); + return res.send(concatenated); + } catch (err) { + console.error('Error serving font:', err); + return res.status(400).header('Content-Type', 'text/plain').send(err); + } }); return app; From b1e1d72f256187a17a740a9ca03323c2eb0743dc Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 00:50:13 -0500 Subject: [PATCH 083/104] Update serve_style.js --- src/serve_style.js | 66 ++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 5dcdbe513..2d4341513 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -91,36 +91,56 @@ export const serve_style = { }); /** - * Handles requests for a font file. - * @param {object} req - Express request object. - * @param {object} res - Express response object. - * @param {string} req.params.fontstack - Name of the font stack. - * @param {string} req.params.range - The range of the font (e.g. 0-255). + * Handles GET requests for sprite images and JSON files. + * @param {express.Request} req - Express request object. + * @param {express.Response} res - Express response object. + * @param {express.NextFunction} next - Express next function. + * @param {string} req.params.id - ID of the sprite. + * @param {string} [req.params.spriteID='default'] - ID of the specific sprite image, defaults to 'default'. + * @param {string} [req.params.scale] - Scale of the sprite image, defaults to ''. + * @param {string} req.params.format - Format of the sprite file, 'png' or 'json'. * @returns {Promise} */ - app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { if (verbose) { - console.log(req.params); - } - const fontstack = decodeURI(req.params.fontstack); - const range = req.params.range; + const { spriteID = 'default', id, format } = req.params; + const spriteScale = allowedSpriteScales(req.params.scale); - try { - const concatenated = await getFontsPbf( - options.serveAllFonts ? null : allowedFonts, - fontPath, - fontstack, - range, - existingFonts, + console.log( + `Handling sprite request for: /styles/%s/sprite/%s%s%s`, + String(id).replace(/\n|\r/g, ''), + String(spriteID).replace(/\n|\r/g, ''), + String(spriteScale).replace(/\n|\r/g, ''), + String(format).replace(/\n|\r/g, ''),, ); + } - res.header('Content-type', 'application/x-protobuf'); - res.header('Last-Modified', lastModified); - return res.send(concatenated); - } catch (err) { - console.error('Error serving font:', err); - return res.status(400).header('Content-Type', 'text/plain').send(err); + const item = repo[id]; + if (!item || !allowedSpriteFormats(format)) { + return res.sendStatus(404); } + + const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); + if (!sprite) { + return res.status(400).send('Bad Sprite ID or Scale'); + } + + const filename = `${sprite.path}${spriteScale}.${format}`; + + // eslint-disable-next-line security/detect-non-literal-fs-filename + fs.readFile(filename, (err, data) => { + if (err) { + console.error('Sprite load error: %s, Error: %s', filename, err); + return res.sendStatus(404); + } + + if (format === 'json') { + res.header('Content-type', 'application/json'); + } else if (format === 'png') { + res.header('Content-type', 'image/png'); + } + return res.send(data); + }); }); return app; From e0574b18877aa380f31c011e55cf9515167a8252 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 00:58:11 -0500 Subject: [PATCH 084/104] Update serve_style.js --- src/serve_style.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 2d4341513..9c7cfc424 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -102,16 +102,16 @@ export const serve_style = { * @returns {Promise} */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { - if (verbose) { - const { spriteID = 'default', id, format } = req.params; - const spriteScale = allowedSpriteScales(req.params.scale); + const { spriteID = 'default', id, format } = req.params; + const spriteScale = allowedSpriteScales(req.params.scale); + if (verbose) { console.log( `Handling sprite request for: /styles/%s/sprite/%s%s%s`, String(id).replace(/\n|\r/g, ''), String(spriteID).replace(/\n|\r/g, ''), String(spriteScale).replace(/\n|\r/g, ''), - String(format).replace(/\n|\r/g, ''),, + String(format).replace(/\n|\r/g, ''), ); } From 96b35a3a8307f216cfa431535ed0a06f5feb11ae Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 01:00:53 -0500 Subject: [PATCH 085/104] Revert "Update serve_style.js" This reverts commit e0574b18877aa380f31c011e55cf9515167a8252. --- src/serve_style.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 9c7cfc424..2d4341513 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -102,16 +102,16 @@ export const serve_style = { * @returns {Promise} */ app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { - const { spriteID = 'default', id, format } = req.params; - const spriteScale = allowedSpriteScales(req.params.scale); - if (verbose) { + const { spriteID = 'default', id, format } = req.params; + const spriteScale = allowedSpriteScales(req.params.scale); + console.log( `Handling sprite request for: /styles/%s/sprite/%s%s%s`, String(id).replace(/\n|\r/g, ''), String(spriteID).replace(/\n|\r/g, ''), String(spriteScale).replace(/\n|\r/g, ''), - String(format).replace(/\n|\r/g, ''), + String(format).replace(/\n|\r/g, ''),, ); } From 54bc82c445781d069654e480fd2926e7f8557c4a Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 01:01:02 -0500 Subject: [PATCH 086/104] Revert "Update serve_style.js" This reverts commit b1e1d72f256187a17a740a9ca03323c2eb0743dc. --- src/serve_style.js | 66 ++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 2d4341513..5dcdbe513 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -91,56 +91,36 @@ export const serve_style = { }); /** - * Handles GET requests for sprite images and JSON files. - * @param {express.Request} req - Express request object. - * @param {express.Response} res - Express response object. - * @param {express.NextFunction} next - Express next function. - * @param {string} req.params.id - ID of the sprite. - * @param {string} [req.params.spriteID='default'] - ID of the specific sprite image, defaults to 'default'. - * @param {string} [req.params.scale] - Scale of the sprite image, defaults to ''. - * @param {string} req.params.format - Format of the sprite file, 'png' or 'json'. + * Handles requests for a font file. + * @param {object} req - Express request object. + * @param {object} res - Express response object. + * @param {string} req.params.fontstack - Name of the font stack. + * @param {string} req.params.range - The range of the font (e.g. 0-255). * @returns {Promise} */ - app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { if (verbose) { - const { spriteID = 'default', id, format } = req.params; - const spriteScale = allowedSpriteScales(req.params.scale); - - console.log( - `Handling sprite request for: /styles/%s/sprite/%s%s%s`, - String(id).replace(/\n|\r/g, ''), - String(spriteID).replace(/\n|\r/g, ''), - String(spriteScale).replace(/\n|\r/g, ''), - String(format).replace(/\n|\r/g, ''),, - ); + console.log(req.params); } + const fontstack = decodeURI(req.params.fontstack); + const range = req.params.range; - const item = repo[id]; - if (!item || !allowedSpriteFormats(format)) { - return res.sendStatus(404); - } + try { + const concatenated = await getFontsPbf( + options.serveAllFonts ? null : allowedFonts, + fontPath, + fontstack, + range, + existingFonts, + ); - const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); - if (!sprite) { - return res.status(400).send('Bad Sprite ID or Scale'); + res.header('Content-type', 'application/x-protobuf'); + res.header('Last-Modified', lastModified); + return res.send(concatenated); + } catch (err) { + console.error('Error serving font:', err); + return res.status(400).header('Content-Type', 'text/plain').send(err); } - - const filename = `${sprite.path}${spriteScale}.${format}`; - - // eslint-disable-next-line security/detect-non-literal-fs-filename - fs.readFile(filename, (err, data) => { - if (err) { - console.error('Sprite load error: %s, Error: %s', filename, err); - return res.sendStatus(404); - } - - if (format === 'json') { - res.header('Content-type', 'application/json'); - } else if (format === 'png') { - res.header('Content-type', 'image/png'); - } - return res.send(data); - }); }); return app; From 3fedd5bb77236ae169386699891308f2a862566e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 01:01:21 -0500 Subject: [PATCH 087/104] Revert "Update serve_style.js" This reverts commit 0f3629c75287a098e2294d30cd57ed5cdb9fc186. --- src/serve_style.js | 115 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 24 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 5dcdbe513..36975ec5d 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -91,36 +91,103 @@ export const serve_style = { }); /** - * Handles requests for a font file. - * @param {object} req - Express request object. - * @param {object} res - Express response object. - * @param {string} req.params.fontstack - Name of the font stack. - * @param {string} req.params.range - The range of the font (e.g. 0-255). + * Handles GET requests for sprite images and JSON files. + * @param {express.Request} req - Express request object. + * @param {express.Response} res - Express response object. + * @param {express.NextFunction} next - Express next function. + * @param {string} req.params.id - ID of the sprite. + * @param {string} [req.params.spriteID='default'] - ID of the specific sprite image, defaults to 'default'. + * @param {string} [req.params.scale] - Scale of the sprite image, defaults to ''. + * @param {string} req.params.format - Format of the sprite file, 'png' or 'json'. * @returns {Promise} */ - app.get('/fonts/:fontstack/:range.pbf', async (req, res) => { + app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { + const { spriteID = 'default', id, format, scale } = req.params; + const sanitizedId = String(id).replace(/\n|\r/g, ''); + const sanitizedScale = scale ? String(scale).replace(/\n|\r/g, '') : ''; + const sanitizedSpriteID = String(spriteID).replace(/\n|\r/g, ''); + const sanitizedFormat = format + ? '.' + String(format).replace(/\n|\r/g, '') + : ''; if (verbose) { - console.log(req.params); - } - const fontstack = decodeURI(req.params.fontstack); - const range = req.params.range; - - try { - const concatenated = await getFontsPbf( - options.serveAllFonts ? null : allowedFonts, - fontPath, - fontstack, - range, - existingFonts, + console.log( + `Handling sprite request for: /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, ); + } + const item = repo[id]; + const validatedFormat = allowedSpriteFormats(format); + if (!item || !validatedFormat) { + if (verbose) + console.error( + `Sprite item or format not found for: /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, + ); + return res.sendStatus(404); + } + const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); + const spriteScale = allowedSpriteScales(scale); + if (!sprite || spriteScale === null) { + if (verbose) + console.error( + `Bad Sprite ID or Scale for: /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, + ); + return res.status(400).send('Bad Sprite ID or Scale'); + } - res.header('Content-type', 'application/x-protobuf'); - res.header('Last-Modified', lastModified); - return res.send(concatenated); - } catch (err) { - console.error('Error serving font:', err); - return res.status(400).header('Content-Type', 'text/plain').send(err); + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if ( + new Date(item.lastModified).getTime() === + new Date(modifiedSince).getTime() + ) { + return res.sendStatus(304); + } } + + const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); + const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; + if (verbose) console.log(`Loading sprite from: %s`, filename); + + // eslint-disable-next-line security/detect-non-literal-fs-filename + fs.readFile(filename, (err, data) => { + if (err) { + if (verbose) + console.error( + 'Sprite load error: %s, Error: %s', + filename, + String(err), + ); + return res.sendStatus(404); + } + + if (validatedFormat === 'json') { + res.header('Content-type', 'application/json'); + } else if (validatedFormat === 'png') { + res.header('Content-type', 'image/png'); + } + if (verbose) + console.log( + `Responding with sprite data for /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, + ); + res.set({ 'Last-Modified': item.lastModified }); + return res.send(data); + }); }); return app; From 1f693003ed3d629d99e7078ff345aef7c2822138 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 01:18:01 -0500 Subject: [PATCH 088/104] Add readFile function --- src/serve_style.js | 148 ++++++++++++++++++++++--------------------- src/utils.js | 154 ++++++++++++++++++++++++--------------------- 2 files changed, 157 insertions(+), 145 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index 36975ec5d..10ef29e21 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -7,7 +7,7 @@ import clone from 'clone'; import express from 'express'; import { validateStyleMin } from '@maplibre/maplibre-gl-style-spec'; -import { fixUrl, allowedOptions } from './utils.js'; +import { fixUrl, allowedOptions, readFile } from './utils.js'; const httpTester = /^https?:\/\//i; const allowedSpriteFormats = allowedOptions(['png', 'json']); @@ -101,94 +101,98 @@ export const serve_style = { * @param {string} req.params.format - Format of the sprite file, 'png' or 'json'. * @returns {Promise} */ - app.get(`/:id/sprite{/:spriteID}{@:scale}{.:format}`, (req, res, next) => { - const { spriteID = 'default', id, format, scale } = req.params; - const sanitizedId = String(id).replace(/\n|\r/g, ''); - const sanitizedScale = scale ? String(scale).replace(/\n|\r/g, '') : ''; - const sanitizedSpriteID = String(spriteID).replace(/\n|\r/g, ''); - const sanitizedFormat = format - ? '.' + String(format).replace(/\n|\r/g, '') - : ''; - if (verbose) { - console.log( - `Handling sprite request for: /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - } - const item = repo[id]; - const validatedFormat = allowedSpriteFormats(format); - if (!item || !validatedFormat) { - if (verbose) - console.error( - `Sprite item or format not found for: /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - return res.sendStatus(404); - } - const sprite = item.spritePaths.find((sprite) => sprite.id === spriteID); - const spriteScale = allowedSpriteScales(scale); - if (!sprite || spriteScale === null) { - if (verbose) - console.error( - `Bad Sprite ID or Scale for: /styles/%s/sprite/%s%s%s`, + app.get( + `/:id/sprite{/:spriteID}{@:scale}{.:format}`, + async (req, res, next) => { + const { spriteID = 'default', id, format, scale } = req.params; + const sanitizedId = String(id).replace(/\n|\r/g, ''); + const sanitizedScale = scale ? String(scale).replace(/\n|\r/g, '') : ''; + const sanitizedSpriteID = String(spriteID).replace(/\n|\r/g, ''); + const sanitizedFormat = format + ? '.' + String(format).replace(/\n|\r/g, '') + : ''; + if (verbose) { + console.log( + `Handling sprite request for: /styles/%s/sprite/%s%s%s`, sanitizedId, sanitizedSpriteID, sanitizedScale, sanitizedFormat, ); - return res.status(400).send('Bad Sprite ID or Scale'); - } + } + const item = repo[id]; + const validatedFormat = allowedSpriteFormats(format); + if (!item || !validatedFormat) { + if (verbose) + console.error( + `Sprite item or format not found for: /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, + ); + return res.sendStatus(404); + } + const sprite = item.spritePaths.find( + (sprite) => sprite.id === spriteID, + ); + const spriteScale = allowedSpriteScales(scale); + if (!sprite || spriteScale === null) { + if (verbose) + console.error( + `Bad Sprite ID or Scale for: /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, + ); + return res.status(400).send('Bad Sprite ID or Scale'); + } - const modifiedSince = req.get('if-modified-since'); - const cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if ( - new Date(item.lastModified).getTime() === - new Date(modifiedSince).getTime() - ) { - return res.sendStatus(304); + const modifiedSince = req.get('if-modified-since'); + const cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if ( + new Date(item.lastModified).getTime() === + new Date(modifiedSince).getTime() + ) { + return res.sendStatus(304); + } } - } - const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); - const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; - if (verbose) console.log(`Loading sprite from: %s`, filename); + const sanitizedSpritePath = sprite.path.replace(/^(\.\.\/)+/, ''); + const filename = `${sanitizedSpritePath}${spriteScale}.${validatedFormat}`; + if (verbose) console.log(`Loading sprite from: %s`, filename); + try { + const data = await readFile(filename); - // eslint-disable-next-line security/detect-non-literal-fs-filename - fs.readFile(filename, (err, data) => { - if (err) { + if (validatedFormat === 'json') { + res.header('Content-type', 'application/json'); + } else if (validatedFormat === 'png') { + res.header('Content-type', 'image/png'); + } if (verbose) + console.log( + `Responding with sprite data for /styles/%s/sprite/%s%s%s`, + sanitizedId, + sanitizedSpriteID, + sanitizedScale, + sanitizedFormat, + ); + res.set({ 'Last-Modified': item.lastModified }); + return res.send(data); + } catch (err) { + if (verbose) { console.error( 'Sprite load error: %s, Error: %s', filename, String(err), ); + } return res.sendStatus(404); } - - if (validatedFormat === 'json') { - res.header('Content-type', 'application/json'); - } else if (validatedFormat === 'png') { - res.header('Content-type', 'image/png'); - } - if (verbose) - console.log( - `Responding with sprite data for /styles/%s/sprite/%s%s%s`, - sanitizedId, - sanitizedSpriteID, - sanitizedScale, - sanitizedFormat, - ); - res.set({ 'Last-Modified': item.lastModified }); - return res.send(data); - }); - }); + }, + ); return app; }, diff --git a/src/utils.js b/src/utils.js index a42c2677f..ea0889538 100644 --- a/src/utils.js +++ b/src/utils.js @@ -185,6 +185,24 @@ export function fixTileJSONCenter(tileJSON) { } } +/** + * Reads a file and returns a Promise with the file data. + * @param {string} filename - Path to the file to read. + * @returns {Promise} - A Promise that resolves with the file data as a Buffer or rejects with an error. + */ +export function readFile(filename) { + return new Promise((resolve, reject) => { + // eslint-disable-next-line security/detect-non-literal-fs-filename + fs.readFile(filename, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); +} + /** * Retrieves font data for a given font and range. * @param {object} allowedFonts - An object of allowed fonts. @@ -194,85 +212,75 @@ export function fixTileJSONCenter(tileJSON) { * @param {object} [fallbacks] - Optional fallback font list. * @returns {Promise} A promise that resolves with the font data Buffer or rejects with an error. */ -function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { - return new Promise((resolve, reject) => { - if (!allowedFonts || (allowedFonts[name] && fallbacks)) { - const fontMatch = name?.match(/^[\w\s-]+$/); - const sanitizedName = fontMatch?.[0] || 'invalid'; - if ( - !name || - typeof name !== 'string' || - name.trim() === '' || - !fontMatch - ) { - console.error( - 'ERROR: Invalid font name: %s', - sanitizedName.replace(/\n|\r/g, ''), - ); - return reject('Invalid font name'); - } +async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { + if (!allowedFonts || (allowedFonts[name] && fallbacks)) { + const fontMatch = name?.match(/^[\w\s-]+$/); + const sanitizedName = fontMatch?.[0] || 'invalid'; + if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { + console.error( + 'ERROR: Invalid font name: %s', + sanitizedName.replace(/\n|\r/g, ''), + ); + throw new Error('Invalid font name'); + } - const rangeMatch = range?.match(/^[\d-]+$/); - const sanitizedRange = rangeMatch?.[0] || 'invalid'; - if (!/^\d+-\d+$/.test(range)) { - console.error( - 'ERROR: Invalid range: %s', - sanitizedRange.replace(/\n|\r/g, ''), - ); - return reject('Invalid range'); - } - const filename = path.join( - fontPath, - sanitizedName, - `${sanitizedRange}.pbf`, + const rangeMatch = range?.match(/^[\d-]+$/); + const sanitizedRange = rangeMatch?.[0] || 'invalid'; + if (!/^\d+-\d+$/.test(range)) { + console.error( + 'ERROR: Invalid range: %s', + sanitizedRange.replace(/\n|\r/g, ''), ); - if (!fallbacks) { - fallbacks = clone(allowedFonts || {}); - } - delete fallbacks[name]; - // eslint-disable-next-line security/detect-non-literal-fs-filename - fs.readFile(filename, (err, data) => { - if (err) { - console.error( - 'ERROR: Font not found: %s, Error: %s', - filename.replace(/\n|\r/g, ''), - String(err), - ); - if (fallbacks && Object.keys(fallbacks).length) { - let fallbackName; + throw new Error('Invalid range'); + } + const filename = path.join( + fontPath, + sanitizedName, + `${sanitizedRange}.pbf`, + ); + + if (!fallbacks) { + fallbacks = clone(allowedFonts || {}); + } + delete fallbacks[name]; + + try { + const data = await readFile(filename); + return data; + } catch (err) { + console.error( + 'ERROR: Font not found: %s, Error: %s', + filename.replace(/\n|\r/g, ''), + String(err), + ); + if (fallbacks && Object.keys(fallbacks).length) { + let fallbackName; - let fontStyle = name.split(' ').pop(); - if (['Regular', 'Bold', 'Italic'].indexOf(fontStyle) < 0) { - fontStyle = 'Regular'; - } - fallbackName = `Noto Sans ${fontStyle}`; - if (!fallbacks[fallbackName]) { - fallbackName = `Open Sans ${fontStyle}`; - if (!fallbacks[fallbackName]) { - fallbackName = Object.keys(fallbacks)[0]; - } - } - console.error( - `ERROR: Trying to use %s as a fallback for: %s`, - fallbackName, - sanitizedName, - ); - delete fallbacks[fallbackName]; - getFontPbf(null, fontPath, fallbackName, range, fallbacks).then( - resolve, - reject, - ); - } else { - reject('Font load error'); + let fontStyle = name.split(' ').pop(); + if (['Regular', 'Bold', 'Italic'].indexOf(fontStyle) < 0) { + fontStyle = 'Regular'; + } + fallbackName = `Noto Sans ${fontStyle}`; + if (!fallbacks[fallbackName]) { + fallbackName = `Open Sans ${fontStyle}`; + if (!fallbacks[fallbackName]) { + fallbackName = Object.keys(fallbacks)[0]; } - } else { - resolve(data); } - }); - } else { - reject('Font not allowed'); + console.error( + `ERROR: Trying to use %s as a fallback for: %s`, + fallbackName, + sanitizedName, + ); + delete fallbacks[fallbackName]; + return getFontPbf(null, fontPath, fallbackName, range, fallbacks); + } else { + throw new Error('Font load error'); + } } - }); + } else { + throw new Error('Font not allowed'); + } } /** * Combines multiple font pbf buffers into one. From 340e5db60ae10f914373aa85a463b010c231d394 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 01:44:32 -0500 Subject: [PATCH 089/104] use readFile, add path.normalize --- src/serve_rendered.js | 22 +++++++++++++++------- src/utils.js | 3 ++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 850953b3f..ac2570fde 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -13,7 +13,6 @@ import '@maplibre/maplibre-gl-native'; // SECTION END import advancedPool from 'advanced-pool'; -import fs from 'node:fs'; import path from 'path'; import url from 'url'; import util from 'util'; @@ -35,6 +34,7 @@ import { fixTileJSONCenter, fetchTileData, allowedOptions, + readFile, } from './utils.js'; import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; import { renderOverlay, renderWatermark, renderAttribution } from './render.js'; @@ -1092,9 +1092,13 @@ export const serve_rendered = { const file = decodeURIComponent(req.url).substring( protocol.length + 3, ); - fs.readFile(path.join(dir, file), (err, data) => { - callback(err, { data: data }); - }); + readFile(path.join(dir, file)) + .then((data) => { + callback(null, { data: data }); + }) + .catch((err) => { + callback(err, null); + }); } else if (protocol === 'fonts') { const parts = req.url.split('/'); const fontstack = decodeURIComponent(parts[2]); @@ -1217,9 +1221,13 @@ export const serve_rendered = { ); } - fs.readFile(file, (err, data) => { - callback(err, { data: data }); - }); + readFile(file) + .then((data) => { + callback(null, { data: data }); + }) + .catch((err) => { + callback(err, null); + }); } else { throw Error( `File does not exist: "${req.url}" - resolved to "${file}"`, diff --git a/src/utils.js b/src/utils.js index ea0889538..b4ede1609 100644 --- a/src/utils.js +++ b/src/utils.js @@ -192,8 +192,9 @@ export function fixTileJSONCenter(tileJSON) { */ export function readFile(filename) { return new Promise((resolve, reject) => { + const sanitizedFilename = path.normalize(filename); // Normalize path, remove .. // eslint-disable-next-line security/detect-non-literal-fs-filename - fs.readFile(filename, (err, data) => { + fs.readFile(String(sanitizedFilename), (err, data) => { if (err) { reject(err); } else { From 7dddbf77d485c60a85dc8f5334dbc3206c175be4 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:02:52 -0500 Subject: [PATCH 090/104] Update serve_rendered.js --- src/serve_rendered.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index ac2570fde..510b99080 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -437,7 +437,7 @@ function calcZForBBox(bbox, w, h, query) { * @param {string} mode Rendering mode ('tile' or 'static'). * @returns {Promise} */ -const respondImage = async ( +async function respondImage( options, item, z, @@ -452,7 +452,7 @@ const respondImage = async ( res, overlay = null, mode = 'tile', -) => { +) { if ( Math.abs(lon) > 180 || Math.abs(lat) > 85.06 || @@ -485,6 +485,7 @@ const respondImage = async ( } else { pool = item.map.renderersStatic[scale]; } + pool.acquire(async (err, renderer) => { // For 512px tiles, use the actual maplibre-native zoom. For 256px tiles, use zoom - 1 let mlglZ; @@ -544,8 +545,8 @@ const respondImage = async ( height: height * scale, }); } - // HACK(Part 2) 256px tiles are a zoom level lower than maplibre-native default tiles. this hack allows tileserver-gl to support zoom 0 256px tiles, which would actually be zoom -1 in maplibre-native. Since zoom -1 isn't supported, a double sized zoom 0 tile is requested and resized here. + if (z === 0 && width === 256) { image.resize(width * scale, height * scale); } @@ -619,7 +620,7 @@ const respondImage = async ( }); }); }); -}; +} /** * Handles requests for tile images. From 62a321262901f4c41dd5b8b419f78be7ea521516 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:27:26 -0500 Subject: [PATCH 091/104] simplify input checking --- src/utils.js | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/utils.js b/src/utils.js index b4ede1609..2e30abf53 100644 --- a/src/utils.js +++ b/src/utils.js @@ -215,31 +215,22 @@ export function readFile(filename) { */ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { - const fontMatch = name?.match(/^[\w\s-]+$/); - const sanitizedName = fontMatch?.[0] || 'invalid'; - if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { - console.error( - 'ERROR: Invalid font name: %s', - sanitizedName.replace(/\n|\r/g, ''), - ); + const sRange = String(range).replace(/\n|\r/g, ''); + const sFontStack = String(name).replace(/\n|\r/g, ''); + if (!sFontStack || name.trim() === '') { + console.error('ERROR: Invalid font name'); throw new Error('Invalid font name'); } - const rangeMatch = range?.match(/^[\d-]+$/); - const sanitizedRange = rangeMatch?.[0] || 'invalid'; - if (!/^\d+-\d+$/.test(range)) { + if (!/^\d+-\d+$/.test(sRange)) { console.error( 'ERROR: Invalid range: %s', sanitizedRange.replace(/\n|\r/g, ''), ); throw new Error('Invalid range'); } - const filename = path.join( - fontPath, - sanitizedName, - `${sanitizedRange}.pbf`, - ); + const filename = path.join(fontPath, sFontStack, `${sRange}.pbf`); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } From 5de617dfe2a2b30e2341cf27940dbb0ec18cced5 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:30:21 -0500 Subject: [PATCH 092/104] Update utils.js --- src/utils.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/utils.js b/src/utils.js index 2e30abf53..a00862ab6 100644 --- a/src/utils.js +++ b/src/utils.js @@ -223,10 +223,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } if (!/^\d+-\d+$/.test(sRange)) { - console.error( - 'ERROR: Invalid range: %s', - sanitizedRange.replace(/\n|\r/g, ''), - ); + console.error('ERROR: Invalid range: %s', sRange); throw new Error('Invalid range'); } @@ -242,7 +239,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } catch (err) { console.error( 'ERROR: Font not found: %s, Error: %s', - filename.replace(/\n|\r/g, ''), + filename, String(err), ); if (fallbacks && Object.keys(fallbacks).length) { @@ -262,7 +259,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { console.error( `ERROR: Trying to use %s as a fallback for: %s`, fallbackName, - sanitizedName, + sFontStack, ); delete fallbacks[fallbackName]; return getFontPbf(null, fontPath, fallbackName, range, fallbacks); From e18874fda0878f678395b50bc6955da01bfd694e Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:34:55 -0500 Subject: [PATCH 093/104] codeql --- src/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.js b/src/utils.js index a00862ab6..614d1460c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -231,7 +231,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } - delete fallbacks[name]; + delete fallbacks[sFontStack]; try { const data = await readFile(filename); @@ -245,7 +245,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (fallbacks && Object.keys(fallbacks).length) { let fallbackName; - let fontStyle = name.split(' ').pop(); + let fontStyle = sFontStack.split(' ').pop(); if (['Regular', 'Bold', 'Italic'].indexOf(fontStyle) < 0) { fontStyle = 'Regular'; } From aceb306f149e4ebff217566ea19202c9fe37daec Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:46:23 -0500 Subject: [PATCH 094/104] Revert "codeql" This reverts commit e18874fda0878f678395b50bc6955da01bfd694e. --- src/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.js b/src/utils.js index 614d1460c..a00862ab6 100644 --- a/src/utils.js +++ b/src/utils.js @@ -231,7 +231,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } - delete fallbacks[sFontStack]; + delete fallbacks[name]; try { const data = await readFile(filename); @@ -245,7 +245,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (fallbacks && Object.keys(fallbacks).length) { let fallbackName; - let fontStyle = sFontStack.split(' ').pop(); + let fontStyle = name.split(' ').pop(); if (['Regular', 'Bold', 'Italic'].indexOf(fontStyle) < 0) { fontStyle = 'Regular'; } From ecfcaeb1a5c577645b12e356c46186f5b02620e1 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:46:31 -0500 Subject: [PATCH 095/104] Revert "Update utils.js" This reverts commit 5de617dfe2a2b30e2341cf27940dbb0ec18cced5. --- src/utils.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/utils.js b/src/utils.js index a00862ab6..2e30abf53 100644 --- a/src/utils.js +++ b/src/utils.js @@ -223,7 +223,10 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } if (!/^\d+-\d+$/.test(sRange)) { - console.error('ERROR: Invalid range: %s', sRange); + console.error( + 'ERROR: Invalid range: %s', + sanitizedRange.replace(/\n|\r/g, ''), + ); throw new Error('Invalid range'); } @@ -239,7 +242,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { } catch (err) { console.error( 'ERROR: Font not found: %s, Error: %s', - filename, + filename.replace(/\n|\r/g, ''), String(err), ); if (fallbacks && Object.keys(fallbacks).length) { @@ -259,7 +262,7 @@ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { console.error( `ERROR: Trying to use %s as a fallback for: %s`, fallbackName, - sFontStack, + sanitizedName, ); delete fallbacks[fallbackName]; return getFontPbf(null, fontPath, fallbackName, range, fallbacks); From 097c0e14552afdd8723d381b4190bed62d2224a9 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 02:46:42 -0500 Subject: [PATCH 096/104] Revert "simplify input checking" This reverts commit 62a321262901f4c41dd5b8b419f78be7ea521516. --- src/utils.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/utils.js b/src/utils.js index 2e30abf53..b4ede1609 100644 --- a/src/utils.js +++ b/src/utils.js @@ -215,22 +215,31 @@ export function readFile(filename) { */ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { - const sRange = String(range).replace(/\n|\r/g, ''); - const sFontStack = String(name).replace(/\n|\r/g, ''); - if (!sFontStack || name.trim() === '') { - console.error('ERROR: Invalid font name'); + const fontMatch = name?.match(/^[\w\s-]+$/); + const sanitizedName = fontMatch?.[0] || 'invalid'; + if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { + console.error( + 'ERROR: Invalid font name: %s', + sanitizedName.replace(/\n|\r/g, ''), + ); throw new Error('Invalid font name'); } - if (!/^\d+-\d+$/.test(sRange)) { + const rangeMatch = range?.match(/^[\d-]+$/); + const sanitizedRange = rangeMatch?.[0] || 'invalid'; + if (!/^\d+-\d+$/.test(range)) { console.error( 'ERROR: Invalid range: %s', sanitizedRange.replace(/\n|\r/g, ''), ); throw new Error('Invalid range'); } + const filename = path.join( + fontPath, + sanitizedName, + `${sanitizedRange}.pbf`, + ); - const filename = path.join(fontPath, sFontStack, `${sRange}.pbf`); if (!fallbacks) { fallbacks = clone(allowedFonts || {}); } From 941e283b81ed94a6b3395ae363de2ac10f548e7d Mon Sep 17 00:00:00 2001 From: acalcutt Date: Sun, 5 Jan 2025 03:10:00 -0500 Subject: [PATCH 097/104] move allowed functions to utils.js --- src/serve_rendered.js | 30 ++++------------------------- src/serve_style.js | 28 ++++++--------------------- src/server.js | 6 +----- src/utils.js | 45 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 53 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 510b99080..af928d993 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -27,13 +27,14 @@ import polyline from '@mapbox/polyline'; import proj4 from 'proj4'; import axios from 'axios'; import { + allowedScales, + allowedTileSizes, getFontsPbf, listFonts, getTileUrls, isValidHttpUrl, fixTileJSONCenter, fetchTileData, - allowedOptions, readFile, } from './utils.js'; import { openPMtiles, getPMtilesInfo } from './pmtiles_adapter.js'; @@ -66,26 +67,6 @@ const httpTester = /^https?:\/\//i; const mercator = new SphericalMercator(); -/** - * Parses a scale string to a number. - * @param {string} scale The scale string (e.g., '2x', '4x'). - * @param {number} maxScaleDigit Maximum allowed scale digit. - * @returns {number|null} The parsed scale as a number or null if invalid. - */ -function parseScale(scale, maxScaleDigit = 9) { - if (scale === undefined) { - return 1; - } - - // eslint-disable-next-line security/detect-non-literal-regexp - const regex = new RegExp(`^[2-${maxScaleDigit}]x$`); - if (!regex.test(scale)) { - return null; - } - - return parseInt(scale.slice(0, -1), 10); -} - mlgl.on('message', (e) => { if (e.severity === 'WARNING' || e.severity === 'ERROR') { console.log('mlgl:', e); @@ -676,13 +657,10 @@ async function handleTileRequest( const z = parseFloat(zParam) | 0; const x = parseFloat(xParam) | 0; const y = parseFloat(yParam) | 0; - const scale = parseScale(scaleParam, maxScaleFactor); + const scale = allowedScales(scaleParam, maxScaleFactor); let parsedTileSize = parseInt(defailtTileSize, 10); if (tileSize) { - const allowedTileSizes = allowedOptions(['256', '512'], { - defaultValue: null, - }); parsedTileSize = parseInt(allowedTileSizes(tileSize), 10); if (parsedTileSize == null) { @@ -778,7 +756,7 @@ async function handleStaticRequest( .send('Invalid width or height provided in size parameter'); } - const scale = parseScale(scaleParam, maxScaleFactor); + const scale = allowedScales(scaleParam, maxScaleFactor); let isRaw = raw === 'raw'; const staticTypeMatch = staticType.match(staticTypeRegex); diff --git a/src/serve_style.js b/src/serve_style.js index 10ef29e21..51b03d3de 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -7,31 +7,15 @@ import clone from 'clone'; import express from 'express'; import { validateStyleMin } from '@maplibre/maplibre-gl-style-spec'; -import { fixUrl, allowedOptions, readFile } from './utils.js'; +import { + allowedSpriteScales, + allowedSpriteFormats, + fixUrl, + readFile, +} from './utils.js'; const httpTester = /^https?:\/\//i; -const allowedSpriteFormats = allowedOptions(['png', 'json']); -/** - * Checks if a string is a valid sprite scale and returns it if it is within the allowed range, and null if it does not conform. - * @param {string} scale - The scale string to validate (e.g., '2x', '3x'). - * @param {number} [maxScale] - The maximum scale value. If no value is passed in, it defaults to a value of 3. - * @returns {string|null} - The valid scale string or null if invalid. - */ -function allowedSpriteScales(scale, maxScale = 3) { - if (!scale) { - return ''; - } - const match = scale?.match(/^([2-9]\d*)x$/); - if (!match) { - return null; - } - const parsedScale = parseInt(match[1], 10); - if (parsedScale <= maxScale) { - return `@${parsedScale}x`; - } - return null; -} export const serve_style = { /** * Initializes the serve_style module. diff --git a/src/server.js b/src/server.js index 524d1ae2b..682e07e9e 100644 --- a/src/server.js +++ b/src/server.js @@ -17,10 +17,10 @@ import { serve_data } from './serve_data.js'; import { serve_style } from './serve_style.js'; import { serve_font } from './serve_font.js'; import { + allowedTileSizes, getTileUrls, getPublicUrl, isValidHttpUrl, - allowedOptions, } from './utils.js'; import { fileURLToPath } from 'url'; @@ -104,10 +104,6 @@ async function start(opts) { ? path.resolve(paths.root, paths.files) : path.resolve(__dirname, '../public/files'); - const allowedTileSizes = allowedOptions(['256', '512'], { - defaultValue: options.tileSize || 256, - }); - const startupPromises = []; for (const type of Object.keys(paths)) { diff --git a/src/utils.js b/src/utils.js index b4ede1609..099acdf20 100644 --- a/src/utils.js +++ b/src/utils.js @@ -8,6 +8,10 @@ import { combine } from '@jsse/pbfont'; import { existsP } from './promises.js'; import { getPMtilesTile } from './pmtiles_adapter.js'; +export const allowedSpriteFormats = allowedOptions(['png', 'json']); + +export const allowedTileSizes = allowedOptions(['256', '512']); + /** * Restrict user input to an allowed set of options. * @param {string[]} opts - An array of allowed option strings. @@ -20,6 +24,47 @@ export function allowedOptions(opts, { defaultValue } = {}) { return (value) => values[value] || defaultValue; } +/** + * Parses a scale string to a number. + * @param {string} scale The scale string (e.g., '2x', '4x'). + * @param {number} maxScale Maximum allowed scale digit. + * @returns {number|null} The parsed scale as a number or null if invalid. + */ +export function allowedScales(scale, maxScale = 9) { + if (scale === undefined) { + return 1; + } + + // eslint-disable-next-line security/detect-non-literal-regexp + const regex = new RegExp(`^[2-${maxScale}]x$`); + if (!regex.test(scale)) { + return null; + } + + return parseInt(scale.slice(0, -1), 10); +} + +/** + * Checks if a string is a valid sprite scale and returns it if it is within the allowed range, and null if it does not conform. + * @param {string} scale - The scale string to validate (e.g., '2x', '3x'). + * @param {number} [maxScale] - The maximum scale value. If no value is passed in, it defaults to a value of 3. + * @returns {string|null} - The valid scale string or null if invalid. + */ +export function allowedSpriteScales(scale, maxScale = 3) { + if (!scale) { + return ''; + } + const match = scale?.match(/^([2-9]\d*)x$/); + if (!match) { + return null; + } + const parsedScale = parseInt(match[1], 10); + if (parsedScale <= maxScale) { + return `@${parsedScale}x`; + } + return null; +} + /** * Replaces local:// URLs with public http(s):// URLs. * @param {object} req - Express request object. From 36cbcf714d4a6eb76d2893ddc6f244baa53e7668 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Thu, 9 Jan 2025 23:28:16 -0500 Subject: [PATCH 098/104] use xy[0],xy[1], --- src/serve_data.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/serve_data.js b/src/serve_data.js index 75cc667d6..d9948e96a 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -237,7 +237,13 @@ export const serve_data = { xy = [minX, minY]; } - const fetchTile = await fetchTileData(source, sourceType, z, x, y); + const fetchTile = await fetchTileData( + source, + sourceType, + z, + xy[0], + xy[1], + ); if (fetchTile == null) return res.status(204).send(); let data = fetchTile.data; From 8f28789002fd488d5f121235e4b2234d3c7b5ce2 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 10 Jan 2025 10:43:54 -0500 Subject: [PATCH 099/104] uprade canvas per https://github.com/maptiler/tileserver-gl/issues/1433 --- package-lock.json | 300 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 2 +- 2 files changed, 290 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index d8d6a3dd3..1332f771e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@sindresorhus/fnv1a": "3.1.0", "advanced-pool": "0.3.3", "axios": "^1.7.7", - "canvas": "2.11.2", + "canvas": "3.0.1", "chokidar": "3.6.0", "clone": "2.1.2", "color": "4.2.3", @@ -2038,6 +2038,25 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/basic-auth": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", @@ -2067,6 +2086,16 @@ "node": ">=8" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/body-parser": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.0.2.tgz", @@ -2145,6 +2174,29 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2252,19 +2304,24 @@ } }, "node_modules/canvas": { - "version": "2.11.2", - "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", - "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-3.0.1.tgz", + "integrity": "sha512-PcpVF4f8RubAeN/jCQQ/UymDKzOiLmRPph8fOTzDnlsUihkO/AUlxuhaa7wGRc3vMcCbV1fzuvyu5cWZlIcn1w==", "hasInstallScript": true, "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", - "nan": "^2.17.0", + "node-addon-api": "^7.0.0", + "prebuild-install": "^7.1.1", "simple-get": "^3.0.3" }, "engines": { - "node": ">=6" + "node": "^18.12.0 || >= 20.9.0" } }, + "node_modules/canvas/node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" + }, "node_modules/chai": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/chai/-/chai-5.1.1.tgz", @@ -2791,6 +2848,14 @@ "node": ">=6" } }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2967,6 +3032,14 @@ "node": ">=0.10.0" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -3537,6 +3610,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/express/-/express-5.0.1.tgz", @@ -3850,6 +3931,11 @@ "node": ">= 0.8" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -4071,6 +4157,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "node_modules/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", @@ -4436,6 +4527,25 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", @@ -5753,6 +5863,11 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/mocha": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.7.3.tgz", @@ -5942,10 +6057,10 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/nan": { - "version": "2.17.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", - "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==" + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" }, "node_modules/natural-compare": { "version": "1.4.0", @@ -6054,6 +6169,17 @@ "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, + "node_modules/node-abi": { + "version": "3.71.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", + "integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-addon-api": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.1.0.tgz", @@ -6755,6 +6881,80 @@ "fflate": "^0.8.0" } }, + "node_modules/prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/prebuild-install/node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/prebuild-install/node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -6841,6 +7041,15 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.2.0.tgz", @@ -6940,6 +7149,33 @@ "node": ">=0.10.0" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", @@ -8163,6 +8399,37 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", @@ -8269,6 +8536,17 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", "devOptional": true }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 0c8dcf320..db21a6221 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "@sindresorhus/fnv1a": "3.1.0", "advanced-pool": "0.3.3", "axios": "^1.7.7", - "canvas": "2.11.2", + "canvas": "3.0.1", "chokidar": "3.6.0", "clone": "2.1.2", "color": "4.2.3", From 930e5719c78c8adb18026729584968be5b2296c7 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 10 Jan 2025 15:15:05 -0500 Subject: [PATCH 100/104] make font regex less restrictive --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 099acdf20..aa1e69156 100644 --- a/src/utils.js +++ b/src/utils.js @@ -260,7 +260,7 @@ export function readFile(filename) { */ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { - const fontMatch = name?.match(/^[\w\s-]+$/); + const fontMatch = name?.match(/^[\p{L}\p{N} \-\.~!*\'()@&=+,#$\[\]]+$/u); const sanitizedName = fontMatch?.[0] || 'invalid'; if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { console.error( From 03b1c31083e905b4abc9d4ffa4f62bccaeed1889 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 10 Jan 2025 15:27:24 -0500 Subject: [PATCH 101/104] fix regex error --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index aa1e69156..5dab80a2a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -260,7 +260,7 @@ export function readFile(filename) { */ async function getFontPbf(allowedFonts, fontPath, name, range, fallbacks) { if (!allowedFonts || (allowedFonts[name] && fallbacks)) { - const fontMatch = name?.match(/^[\p{L}\p{N} \-\.~!*\'()@&=+,#$\[\]]+$/u); + const fontMatch = name?.match(/^[\p{L}\p{N} \-\.~!*'()@&=+,#$\[\]]+$/u); const sanitizedName = fontMatch?.[0] || 'invalid'; if (!name || typeof name !== 'string' || name.trim() === '' || !fontMatch) { console.error( From 1df0550481047a7dfbf9546fb79fd9ad802a3cbe Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 10 Jan 2025 19:22:44 -0500 Subject: [PATCH 102/104] Add version and changelog --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 108af87df..be2fed332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # tileserver-gl changelog +## 5.1.0-pre.0 +* Upgrade Express to v5 +Canvas v3 + code cleanup (https://github.com/maptiler/tileserver-gl/pull/1429) by @acalcutt +* Terrain Preview and simple Elevation Query (https://github.com/maptiler/tileserver-gl/pull/1425 and #https://github.com/maptiler/tileserver-gl/pull/1432) by @okimiko +* add progressive rendering option for static jpeg images (#1397) by @samuel-git + ## 5.0.0 * Update Maplibre-Native to [v6.0.0](https://github.com/maplibre/maplibre-native/releases/tag/node-v6.0.0) release by @acalcutt in https://github.com/maptiler/tileserver-gl/pull/1376 and @dependabot in https://github.com/maptiler/tileserver-gl/pull/1381 * This first release that use Metal for rendering instead of OpenGL (ES) for macOS. diff --git a/package-lock.json b/package-lock.json index b94d941a3..7fcd23e53 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "tileserver-gl", - "version": "5.0.0", + "version": "5.1.0-pre.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "tileserver-gl", - "version": "5.0.0", + "version": "5.1.0-pre.0", "license": "BSD-2-Clause", "dependencies": { "@jsse/pbfont": "^0.2.2", diff --git a/package.json b/package.json index 888370f29..3d40623a9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tileserver-gl", - "version": "5.0.0", + "version": "5.1.0-pre.0", "description": "Map tile server for JSON GL styles - vector and server side generated raster tiles", "main": "src/main.js", "bin": "src/main.js", From 38d64a76135f472d7da8ee35b148fb6964f1a3be Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 10 Jan 2025 19:23:33 -0500 Subject: [PATCH 103/104] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be2fed332..c1636fb20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 5.1.0-pre.0 * Upgrade Express to v5 +Canvas v3 + code cleanup (https://github.com/maptiler/tileserver-gl/pull/1429) by @acalcutt -* Terrain Preview and simple Elevation Query (https://github.com/maptiler/tileserver-gl/pull/1425 and #https://github.com/maptiler/tileserver-gl/pull/1432) by @okimiko +* Terrain Preview and simple Elevation Query (https://github.com/maptiler/tileserver-gl/pull/1425 and https://github.com/maptiler/tileserver-gl/pull/1432) by @okimiko * add progressive rendering option for static jpeg images (#1397) by @samuel-git ## 5.0.0 From fde1b037bbe5a46234984e732c30ef045ebf6b02 Mon Sep 17 00:00:00 2001 From: acalcutt Date: Fri, 10 Jan 2025 19:26:21 -0500 Subject: [PATCH 104/104] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1636fb20..8508880cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # tileserver-gl changelog ## 5.1.0-pre.0 -* Upgrade Express to v5 +Canvas v3 + code cleanup (https://github.com/maptiler/tileserver-gl/pull/1429) by @acalcutt +* Upgrade Express to v5 + Canvas to v3 + code cleanup (https://github.com/maptiler/tileserver-gl/pull/1429) by @acalcutt * Terrain Preview and simple Elevation Query (https://github.com/maptiler/tileserver-gl/pull/1425 and https://github.com/maptiler/tileserver-gl/pull/1432) by @okimiko * add progressive rendering option for static jpeg images (#1397) by @samuel-git