Made project use a callback provider to avoid letting almost the whole world on blank
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
# Included inside nginx's http block by the stock nginx.conf.
|
||||
proxy_cache_path /var/cache/nginx/callback-provider levels=1:2 keys_zone=callback_provider:10m inactive=7d max_size=2g use_temp_path=off;
|
||||
+18
-1
@@ -8,10 +8,27 @@ server {
|
||||
|
||||
# Public API is x/y/z. try_files performs an internal lookup at z/x/y.png.
|
||||
location ~ ^/tiles/(?<tile_x>[0-9]+)/(?<tile_y>[0-9]+)/(?<tile_z>[0-9]+)$ {
|
||||
try_files /tiles/$tile_z/$tile_x/$tile_y.png =404;
|
||||
if ($request_method = OPTIONS) {
|
||||
add_header Access-Control-Allow-Origin "*" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept" always;
|
||||
add_header Content-Length 0;
|
||||
return 204;
|
||||
}
|
||||
try_files /tiles/$tile_z/$tile_x/$tile_y.png @callback_provider_tile;
|
||||
add_header Cache-Control "public, max-age=86400, stale-while-revalidate=604800" always;
|
||||
add_header Access-Control-Allow-Origin "*" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept" always;
|
||||
}
|
||||
|
||||
# A callback-provider base layer fills transparent edges when configured.
|
||||
location ~ ^/base/(?<base_x>[0-9]+)/(?<base_y>[0-9]+)/(?<base_z>[0-9]+)$ {
|
||||
try_files /__callback_provider_base__ @callback_provider_base;
|
||||
}
|
||||
|
||||
include /etc/nginx/includes/callback-provider.conf;
|
||||
|
||||
# metadata.json and internal z/x/y.png paths are intentionally not public.
|
||||
location / {
|
||||
return 404;
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
python3 /usr/local/bin/callback-provider.py --nginx-output /etc/nginx/includes/callback-provider.conf
|
||||
+37
-13
@@ -21,6 +21,11 @@ from xml.etree import ElementTree as ET
|
||||
|
||||
from PIL import Image
|
||||
|
||||
SCRIPT_DIRECTORY = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIRECTORY) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIRECTORY))
|
||||
from callback_provider import ProviderValidationError, validate_callback_provider
|
||||
|
||||
|
||||
PIPELINE_VERSION = "1.0.0"
|
||||
WORLD_WIDTH_METERS = 40075016.68557849
|
||||
@@ -147,7 +152,8 @@ def read_config(path: Path) -> dict:
|
||||
|
||||
|
||||
def effective_zooms(auto_max: int, relative_source: str, config: dict) -> tuple[int, int, bool]:
|
||||
auto_min = max(6, min(10, auto_max - 7))
|
||||
# Low zooms are cheap (usually one tile per source) and make isolated maps discoverable from a world view.
|
||||
auto_min = 0
|
||||
override = config["sources"].get(relative_source, {})
|
||||
minimum = override.get("minzoom")
|
||||
maximum = override.get("maxzoom")
|
||||
@@ -364,6 +370,10 @@ def compose_layer(source_tiles: Path, final_tiles: Path) -> None:
|
||||
result.save(destination, format="PNG", optimize=False)
|
||||
|
||||
|
||||
def write_empty_tile(tiles_dir: Path) -> None:
|
||||
Image.new("RGBA", (256, 256), (255, 255, 255, 255)).save(tiles_dir / "empty.png", format="PNG", optimize=False)
|
||||
|
||||
|
||||
def tile_overlay(overlay: dict, work_root: Path, final_tiles: Path, processes: int) -> None:
|
||||
layer_root = work_root / f"layer-{overlay['priority']:04d}-{overlay['index']}"
|
||||
help_result = subprocess.run(["gdal2tiles.py", "--help"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
@@ -376,16 +386,21 @@ def tile_overlay(overlay: dict, work_root: Path, final_tiles: Path, processes: i
|
||||
shutil.rmtree(layer_root, ignore_errors=True)
|
||||
|
||||
|
||||
def publish(stage: Path, data_dir: Path) -> None:
|
||||
current = data_dir / "tiles"
|
||||
backup = data_dir / ".tiles-previous"
|
||||
if backup.exists():
|
||||
shutil.rmtree(backup)
|
||||
if current.exists():
|
||||
current.rename(backup)
|
||||
stage.rename(current)
|
||||
if backup.exists():
|
||||
shutil.rmtree(backup)
|
||||
def remove_generated_path(path: Path) -> None:
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink()
|
||||
|
||||
|
||||
def publish(stage: Path, tiles_dir: Path) -> None:
|
||||
"""Publish staged tiles into a directory which may be a bind mount."""
|
||||
tiles_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in tiles_dir.iterdir():
|
||||
remove_generated_path(existing)
|
||||
for generated in stage.iterdir():
|
||||
shutil.move(str(generated), str(tiles_dir / generated.name))
|
||||
stage.rmdir()
|
||||
|
||||
|
||||
def source_record(descriptor: dict, config: dict) -> dict:
|
||||
@@ -415,8 +430,11 @@ def source_record(descriptor: dict, config: dict) -> dict:
|
||||
|
||||
|
||||
def build(arguments: argparse.Namespace) -> None:
|
||||
# Validate before reading or warping any source, so a malformed provider never wastes a full build.
|
||||
validate_callback_provider(os.environ.get("CALLBACK_PROVIDER"))
|
||||
source_dir = Path(arguments.source_dir).resolve()
|
||||
data_dir = Path(arguments.data_dir).resolve()
|
||||
tiles_dir = Path(arguments.tiles_dir).resolve()
|
||||
config = read_config(Path(arguments.config).resolve())
|
||||
if not source_dir.is_dir():
|
||||
raise RuntimeError(f"diretório source inexistente: {source_dir}")
|
||||
@@ -468,13 +486,15 @@ def build(arguments: argparse.Namespace) -> None:
|
||||
"generated_at": datetime.now(UTC).isoformat(),
|
||||
"minzoom": global_min,
|
||||
"maxzoom": global_max,
|
||||
"tile_background": "transparent",
|
||||
"processed_sources": records,
|
||||
"sources_with_error": errors,
|
||||
"source_count": len(records),
|
||||
"error_count": len(errors),
|
||||
}
|
||||
write_empty_tile(stage)
|
||||
(stage / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
publish(stage, data_dir)
|
||||
publish(stage, tiles_dir)
|
||||
except Exception:
|
||||
shutil.rmtree(stage, ignore_errors=True)
|
||||
raise
|
||||
@@ -486,10 +506,14 @@ def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-dir", default=os.environ.get("SOURCE_DIR", "source"))
|
||||
parser.add_argument("--data-dir", default=os.environ.get("DATA_DIR", "."))
|
||||
parser.add_argument("--tiles-dir", default=os.environ.get("TILES_DIR"))
|
||||
parser.add_argument("--config", default=os.environ.get("MAPS_CONFIG", "config/maps.json"))
|
||||
parser.add_argument("--source-filter", action="append", help="caminho relativo exato sob source/; repetível")
|
||||
parser.add_argument("--processes", type=int, default=max(1, int(os.environ.get("GDAL2TILES_PROCESSES", "1"))))
|
||||
return parser.parse_args()
|
||||
arguments = parser.parse_args()
|
||||
if arguments.tiles_dir is None:
|
||||
arguments.tiles_dir = str(Path(arguments.data_dir) / "tiles")
|
||||
return arguments
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate CALLBACK_PROVIDER and render its safe Nginx proxy locations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
class ProviderValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CallbackProvider:
|
||||
origin: str
|
||||
path_template: str
|
||||
|
||||
|
||||
def validate_callback_provider(value: str | None) -> CallbackProvider | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if value != value.strip() or len(value) > 2048 or any(character.isspace() for character in value):
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER não pode conter espaços e deve ter até 2048 caracteres")
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER deve ser uma URL HTTPS pública, sem usuário ou senha")
|
||||
if parsed.query or parsed.fragment or not parsed.path.startswith("/"):
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER não aceita query, fragmento e exige um caminho absoluto")
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as error:
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER possui porta inválida") from error
|
||||
host = parsed.hostname
|
||||
if not host or not re.fullmatch(r"[A-Za-z0-9.-]+", host) or ".." in host or host.startswith(".") or host.endswith("."):
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER possui host inválido")
|
||||
try:
|
||||
address = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
address = None
|
||||
if address is not None and not address.is_global:
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER não pode apontar para endereço IP privado ou reservado")
|
||||
if port is not None and not 1 <= port <= 65535:
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER possui porta inválida")
|
||||
if not re.fullmatch(r"/[A-Za-z0-9._~/%{}-]+", parsed.path) or "/../" in f"{parsed.path}/" or any(token in parsed.path for token in ("$", ";", "\\")):
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER possui caminho não permitido")
|
||||
for token in ("{x}", "{y}", "{z}"):
|
||||
if parsed.path.count(token) != 1:
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER deve conter exatamente uma ocorrência de {x}, {y} e {z}")
|
||||
remaining_placeholders = parsed.path
|
||||
for token in ("{x}", "{y}", "{z}"):
|
||||
remaining_placeholders = remaining_placeholders.replace(token, "")
|
||||
if "{" in remaining_placeholders or "}" in remaining_placeholders:
|
||||
raise ProviderValidationError("CALLBACK_PROVIDER contém placeholder desconhecido")
|
||||
return CallbackProvider(origin=f"https://{parsed.netloc}", path_template=parsed.path)
|
||||
|
||||
|
||||
def render_location(name: str, provider: CallbackProvider | None, x: str, y: str, z: str, label: str) -> str:
|
||||
headers = """ add_header Cache-Control \"public, max-age=86400, stale-while-revalidate=604800\" always;
|
||||
add_header Access-Control-Allow-Origin \"*\" always;
|
||||
add_header Access-Control-Allow-Methods \"GET, HEAD, OPTIONS\" always;
|
||||
add_header Access-Control-Allow-Headers \"Origin, Range, Accept\" always;"""
|
||||
if provider is None:
|
||||
return f"""location @{name} {{
|
||||
try_files /tiles/empty.png =500;
|
||||
{headers}
|
||||
}}
|
||||
"""
|
||||
rendered_path = provider.path_template.replace("{x}", f"${x}").replace("{y}", f"${y}").replace("{z}", f"${z}")
|
||||
return f"""location @{name} {{
|
||||
rewrite ^ {rendered_path} break;
|
||||
proxy_pass {provider.origin};
|
||||
proxy_set_header Host $proxy_host;
|
||||
proxy_ssl_server_name on;
|
||||
proxy_cache callback_provider;
|
||||
proxy_cache_valid 200 7d;
|
||||
proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
|
||||
add_header X-Map-Source \"Callback provider {label}\" always;
|
||||
{headers}
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def render_nginx(provider: CallbackProvider | None) -> str:
|
||||
return (
|
||||
"# Generated at container startup. Do not edit.\n"
|
||||
+ render_location("callback_provider_tile", provider, "tile_x", "tile_y", "tile_z", "fallback")
|
||||
+ "\n"
|
||||
+ render_location("callback_provider_base", provider, "base_x", "base_y", "base_z", "base")
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--nginx-output", type=Path)
|
||||
arguments = parser.parse_args()
|
||||
provider = validate_callback_provider(__import__("os").environ.get("CALLBACK_PROVIDER"))
|
||||
if arguments.nginx_output:
|
||||
arguments.nginx_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.nginx_output.write_text(render_nginx(provider), encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except ProviderValidationError as error:
|
||||
raise SystemExit(f"CALLBACK_PROVIDER inválido: {error}")
|
||||
@@ -35,6 +35,6 @@ done
|
||||
headers="$(curl -sS -D - -o /dev/null "$base_url/tiles/$x/$y/$z")"
|
||||
printf '%s\n' "$headers" | grep -qi '^HTTP/.* 200'
|
||||
printf '%s\n' "$headers" | grep -qi '^Content-Type: image/png'
|
||||
test "$(curl -sS -o /dev/null -w '%{http_code}' "$base_url/tiles/0/0/0")" = "404"
|
||||
printf '%s\n' "$headers" | grep -qi '^Access-Control-Allow-Origin: \*'
|
||||
|
||||
printf 'Smoke test passou: /tiles/%s/%s/%s retornou PNG e tile descoberta retornou 404.\n' "$x" "$y" "$z"
|
||||
printf 'Smoke test passou: /tiles/%s/%s/%s retornou PNG/CORS.\n' "$x" "$y" "$z"
|
||||
|
||||
@@ -26,12 +26,51 @@ class BuildTilesTests(unittest.TestCase):
|
||||
config = {"global": {"minzoom": 7, "maxzoom": 12}, "sources": {"Altis.kmz": {"minzoom": 9, "maxzoom": 14}}}
|
||||
self.assertEqual(build_tiles.effective_zooms(11, "Altis.kmz", config), (9, 14, True))
|
||||
|
||||
def test_callback_provider_accepts_safe_https_xyz_template(self):
|
||||
provider = build_tiles.validate_callback_provider("https://tiles.example.org/maps/{z}/{x}/{y}.png")
|
||||
self.assertEqual(provider.origin, "https://tiles.example.org")
|
||||
|
||||
def test_callback_provider_rejects_missing_or_unsafe_template(self):
|
||||
with self.assertRaises(build_tiles.ProviderValidationError):
|
||||
build_tiles.validate_callback_provider("http://tiles.example.org/{z}/{x}/{y}.png")
|
||||
with self.assertRaises(build_tiles.ProviderValidationError):
|
||||
build_tiles.validate_callback_provider("https://127.0.0.1/{z}/{x}/{y}.png")
|
||||
with self.assertRaises(build_tiles.ProviderValidationError):
|
||||
build_tiles.validate_callback_provider("https://tiles.example.org/{z}/{x}.png")
|
||||
|
||||
def test_config_normalizes_source_key(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "maps.json"
|
||||
path.write_text(json.dumps({"global": {}, "sources": {"nested\\map.kmz": {"minzoom": 8}}}))
|
||||
self.assertIn("nested/map.kmz", build_tiles.read_config(path)["sources"])
|
||||
|
||||
def test_publish_replaces_generated_output_directory(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
stage = root / "stage"
|
||||
output = root / "tiles"
|
||||
stage.mkdir()
|
||||
output.mkdir()
|
||||
(stage / "metadata.json").write_text("new")
|
||||
(stage / "8").mkdir()
|
||||
(stage / "8" / "tile.png").write_text("tile")
|
||||
(output / "stale.png").write_text("stale")
|
||||
build_tiles.publish(stage, output)
|
||||
self.assertFalse((output / "stale.png").exists())
|
||||
self.assertEqual((output / "metadata.json").read_text(), "new")
|
||||
self.assertTrue((output / "8" / "tile.png").is_file())
|
||||
|
||||
def test_empty_tile_is_opaque_white_png(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
target = Path(directory)
|
||||
build_tiles.write_empty_tile(target)
|
||||
with build_tiles.Image.open(target / "empty.png") as image:
|
||||
self.assertEqual(image.mode, "RGBA")
|
||||
self.assertEqual(image.size, (256, 256))
|
||||
self.assertEqual(image.getpixel((0, 0)), (255, 255, 255, 255))
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user